diff options
| author | jianlong <[email protected]> | 2021-08-19 17:26:25 +0800 |
|---|---|---|
| committer | jianlong <[email protected]> | 2021-08-19 17:26:25 +0800 |
| commit | 0dc57dfdd75474c7dad28628e6f87e2175678ff2 (patch) | |
| tree | c9ceb2d225a5a9200441c90b3b790bf0c756d03f | |
基础版本v21.07
540 files changed, 54647 insertions, 0 deletions
diff --git a/.idea/Galaxy-auto-deploy-cluster-21.07.iml b/.idea/Galaxy-auto-deploy-cluster-21.07.iml new file mode 100644 index 0000000..0abfce7 --- /dev/null +++ b/.idea/Galaxy-auto-deploy-cluster-21.07.iml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8"?> +<module type="PYTHON_MODULE" version="4"> + <component name="NewModuleRootManager"> + <content url="file://$MODULE_DIR$" /> + <orderEntry type="inheritedJdk" /> + <orderEntry type="sourceFolder" forTests="false" /> + </component> + <component name="TemplatesService"> + <option name="TEMPLATE_FOLDERS"> + <list> + <option value="$MODULE_DIR$/parcels/roles/init/druid/templates" /> + </list> + </option> + </component> + <component name="TestRunnerService"> + <option name="PROJECT_TEST_RUNNER" value="Unittests" /> + </component> +</module>
\ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..15a15b2 --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,4 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="Encoding" addBOMForNewFiles="with NO BOM" /> +</project>
\ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..9754207 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="JavaScriptSettings"> + <option name="languageLevel" value="ES6" /> + </component> + <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (python3.7)" project-jdk-type="Python SDK" /> +</project>
\ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..8e5cae9 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="ProjectModuleManager"> + <modules> + <module fileurl="file://$PROJECT_DIR$/.idea/Galaxy-auto-deploy-cluster-21.07.iml" filepath="$PROJECT_DIR$/.idea/Galaxy-auto-deploy-cluster-21.07.iml" /> + </modules> + </component> +</project>
\ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="VcsDirectoryMappings"> + <mapping directory="$PROJECT_DIR$" vcs="Git" /> + </component> +</project>
\ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..52d3c72 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,102 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="ChangeListManager"> + <list default="true" id="55c8c5b9-9b57-431e-a0ed-0064b85979fa" name="Default Changelist" comment="" /> + <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" /> + <option name="SHOW_DIALOG" value="false" /> + <option name="HIGHLIGHT_CONFLICTS" value="true" /> + <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" /> + <option name="LAST_RESOLUTION" value="IGNORE" /> + </component> + <component name="Git.Settings"> + <option name="ROOT_SYNC" value="DONT_SYNC" /> + </component> + <component name="ProjectFrameBounds" extendedState="6" fullScreen="true"> + <option name="width" value="2560" /> + <option name="height" value="1440" /> + </component> + <component name="ProjectView"> + <navigator proportions="" version="1"> + <foldersAlwaysOnTop value="true" /> + </navigator> + <panes> + <pane id="ProjectPane"> + <subPane> + <expand> + <path> + <item name="Galaxy-auto-deploy-cluster-21.07" type="b2602c69:ProjectViewProjectNode" /> + <item name="Galaxy-auto-deploy-cluster-21.07" type="462c0819:PsiDirectoryNode" /> + </path> + </expand> + <select /> + </subPane> + </pane> + <pane id="Scope" /> + </panes> + </component> + <component name="PropertiesComponent"> + <property name="WebServerToolWindowFactoryState" value="false" /> + <property name="last_opened_file_path" value="$PROJECT_DIR$" /> + <property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" /> + <property name="nodejs_npm_path_reset_for_default_project" value="true" /> + <property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" /> + </component> + <component name="RunDashboard"> + <option name="ruleStates"> + <list> + <RuleState> + <option name="name" value="ConfigurationTypeDashboardGroupingRule" /> + </RuleState> + <RuleState> + <option name="name" value="StatusDashboardGroupingRule" /> + </RuleState> + </list> + </option> + </component> + <component name="SvnConfiguration"> + <configuration /> + </component> + <component name="TaskManager"> + <task active="true" id="Default" summary="Default task"> + <changelist id="55c8c5b9-9b57-431e-a0ed-0064b85979fa" name="Default Changelist" comment="" /> + <created>1629364019161</created> + <option name="number" value="Default" /> + <option name="presentableId" value="Default" /> + <updated>1629364019161</updated> + <workItem from="1629364020595" duration="323000" /> + </task> + <servers /> + </component> + <component name="TimeTrackingManager"> + <option name="totallyTimeSpent" value="323000" /> + </component> + <component name="ToolWindowManager"> + <frame x="0" y="0" width="2560" height="1440" extended-state="6" /> + <layout> + <window_info id="Favorites" side_tool="true" /> + <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.24980143" /> + <window_info id="Structure" order="1" side_tool="true" weight="0.25" /> + <window_info anchor="bottom" id="Docker" show_stripe_button="false" /> + <window_info anchor="bottom" id="Database Changes" /> + <window_info anchor="bottom" id="Version Control" /> + <window_info anchor="bottom" id="Python Console" /> + <window_info anchor="bottom" id="Terminal" /> + <window_info anchor="bottom" id="Event Log" side_tool="true" /> + <window_info anchor="bottom" id="Message" order="0" /> + <window_info anchor="bottom" id="Find" order="1" /> + <window_info anchor="bottom" id="Run" order="2" /> + <window_info anchor="bottom" id="Debug" order="3" weight="0.4" /> + <window_info anchor="bottom" id="Cvs" order="4" weight="0.25" /> + <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" /> + <window_info anchor="bottom" id="TODO" order="6" /> + <window_info anchor="right" id="SciView" /> + <window_info anchor="right" id="Database" /> + <window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" /> + <window_info anchor="right" id="Ant Build" order="1" weight="0.25" /> + <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" /> + </layout> + </component> + <component name="TypeScriptGeneratedFilesManager"> + <option name="version" value="1" /> + </component> +</project>
\ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..2831d45 --- /dev/null +++ b/README.md @@ -0,0 +1,79 @@ +# 使用手册 + +[TOC] + +## 目录结构 +Galaxy-auto-deploy-cluster-21.07 +├── inventories +│ └── pro +│ ├── group_vars +│ └── hosts +├── parcels +│ ├── menu +│ │ ├── appMenu +│ │ ├── commonMenu +│ │ └── homePage +│ ├── roles +│ │ ├── check +│ │ ├── components +│ │ ├── dependencies +│ │ ├── init +│ │ └── services +│ ├── Arangodb.yml +│ ├── check-components.yml +│ ├── check-services.yml +│ ├── Clickhouse.yml +│ ├── Druid.yml +│ ├── environment.yml +│ ├── galaxy-chproxy.yml +│ ├── galaxy-gateway-keepalive.yml +│ ├── galaxy-gateway-nginx.yml +│ ├── galaxy-gohangout.yml +│ ├── galaxy-hos-keepalive.yml +│ ├── galaxy-hos-nginx.yml +│ ├── galaxy-hos-service.yml +│ ├── galaxy-job-service.yml +│ ├── galaxy-qgw-service.yml +│ ├── galaxy-report-service.yml +│ ├── Hadoop.yml +│ ├── HBase.yml +│ ├── init-clickhouse.yml +│ ├── init-druid.yml +│ ├── init-galaxy-gateway-keepalive.yml +│ ├── init-galaxy-hos-service.yml +│ ├── init-galaxy-job-service.yml +│ ├── init-hbase.yml +│ ├── init-kafka.yml +│ ├── init-spark.yml +│ ├── init-storm.yml +│ ├── Kafka.yml +│ ├── Mariadb.yml +│ ├── Nacos.yml +│ ├── Spark.yml +│ ├── Storm.yml +│ ├── test1.yml +│ ├── test.yml +│ └── Zookeeper.yml +├── software_packages + + +## 纪要 +- 主机ip配置文件 +inventories/pro/hosts + +- 所有role(主机)使用的变量 +inventories/pro/group_vars/all +新开发的role的变量可定义到此文件中统一管理, 已存在的role的变量直接修改此文件即可 +`注意`: 新增变量名不可与已有变量发生重复 + +- 部署包目录 +software_packages +为方便管理, 将相关部署包和镜像移到software_packages文件夹下(避免大文件上传gitlab) + + +## 使用说明 +1. 配置 inventories/pro/hosts 部署组件的主机 + +2. 视情况修改 inventories/pro/group_vars/all 里的相关配置 + +3. 确实好配置后执行 "sh install.sh" 进行安装 diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..e5c22f9 --- /dev/null +++ b/install.sh @@ -0,0 +1,666 @@ +#!/bin/bash + +NUM_DIR="/usr/galaxyinstmp" +if [ ! -d "$NUM_DIR/" ];then + mkdir -p $NUM_DIR/ +fi + +cd parcels + +function checkManager() { +echo "Check whether the current executor has Docker and Ansible environment, if not, install it" +echo -e "\033[31;1mThe process may prompt command not found error, please ignore it.\033[0m" + +sleep 2 +a=`docker -v` +hasDocker=$? +if [[ $hasDocker != '0' ]]; then + echo " + " + echo -e "\033[31;1mThis Docker not installed, Please install\033[0m" + exit 1 +else + echo " + " + echo -e "\033[32;1mThis Docker installed, the version is:\033[0m" + docker -v + +fi + +a=`ansible --version` +hasAnsible=$? +if [[ $hasAnsible != '0' ]]; then + echo " + " + echo -e "\033[31;1mThis Ansible not installed, Please install\033[0m" + exit 1 +else + echo " + " + echo -e "\033[32;1mThis Ansible installed, the version is:\033[0m" + ansible --version + echo 1 > $NUM_DIR/checkManager +fi +sleep 5 +} + + +function checkCluster() { +clear +echo "Check the environment to be deployed, including JDK, PIP, Docker, Firewall, TimeZone if not installed" +sleep 2 +ansible-playbook -i ../inventories/pro/hosts environment.yml +if [ $? -eq '0' ];then + clear + echo -e "\033[32;1mEnvironmental check complete, ready for installation\033[0m" + sleep 5 + echo 1 > $NUM_DIR/checkCluster +else + exit 1 +fi +} + + +#记录组件安装次数 +function recordinstnum() { +FILE_NAME=$1 + +if [ -f "$NUM_DIR/$FILE_NAME" ];then + OLD_NUM=`cat $NUM_DIR/$FILE_NAME` + INS_NUM=`expr $OLD_NUM + 1` + echo $INS_NUM > $NUM_DIR/$FILE_NAME +else + echo 1 > $NUM_DIR/$FILE_NAME +fi +} + +#获取组件安装次数 +function getrecordnum() { +FILE_NAME=$1 + +if [ ! -d "$NUM_DIR/" ];then + mkdir -p $NUM_DIR/ +fi + +if [ -f "$NUM_DIR/$FILE_NAME" ];then + echo `cat $NUM_DIR/$FILE_NAME` +else + echo 0 > $NUM_DIR/$FILE_NAME + echo 0 +fi +} + +#输出组件安装信息 +function echoSuccess() { +NAME=$1 + +echo -e " +\033[32;1m$NAME Components have been installed.\033[0m +" +echo "`date "+%Y-%m-%d %H:%M:%S"` - $NAME 剧本已执行完成." >> ../auditlog +sleep 5 +clear +} + +#输出组件安装信息 +function echoError() { +NAME=$1 + +echo -e "\033[31;1m $NAME The installation fails. Please check\033[0m" +echo "`date "+%Y-%m-%d %H:%M:%S"` - $NAME 剧本执行异常." >> ../auditlog +sleep 5 +} + +#=========================================commons======================================# +#单个安装执行方法，根据传入的组件名称进行安装 +#若安装出现异常导致ansible停止，则脚本直接终止 +#若本机多次安装过某组件，则会提示是否继续安装；y 继续 N 停止 +function installation() { +NAME=$1 + +is=`getrecordnum $NAME` +if [ $is -lt '1' ];then + ansible-playbook -i ../inventories/pro/hosts $NAME.yml + if [ $? -eq '0' ];then + sleep 10 + echoSuccess $NAME + recordinstnum $NAME + else + echoError $NAME + exit 1 + fi +else + clear && echo -e "\033[31;1m本执行机已成功安装过 $is 次$NAME，缺是否继续安装，误操作可根据提示返回 \033[0m" && sleep 1 + while true; do + read -p "y/Y 继续执行安装,n/N 返回选择菜单：" yn + case $yn in + [Yy]*) + ansible-playbook -i ../inventories/pro/hosts $NAME.yml + if [ $? -eq '0' ];then + sleep 10 + echoSuccess $NAME + recordinstnum $NAME + break + else + echoError $NAME + exit 1 + fi + ;; + [Nn]*) + break;; + * ) + echo "Please answer yes or no." + ;; + esac +done +fi + +} + +#组合安装正常逻辑，逐个组件询问，Y执行 N不执行安装。 +#记录数组通过循环顺序执行安装 +function installAllCommon() { +clear + +i=0 +ins_names=(Zookeeper Mariadb galaxy-gateway-keepalive Nacos Kafka Storm Hadoop HBase Clickhouse Druid Spark Arangodb) +echo ${ins_names[@]} > $NUM_DIR/common_list + +#根据组合进行顺序安装 +for element in ${ins_names[@]} +do + ansible-playbook -i ../inventories/pro/hosts $element.yml + if [ $? -eq '0' ];then + sleep 10 + echoSuccess $element + recordinstnum $element + unset ins_names[$i] + let i++ + echo ${ins_names[@]} > $NUM_DIR/common_list + else + echoError $element + exit 1 + fi +done +} + +#组合安装异常终止执行操作 +#询问是否需要再上次基础上执行，否则清空文件跳出循环 +function abnormalCombination() { +clear +while true; do +echo -e " +\033[31;1m上次选择部署后还有 `cat $NUM_DIR/common_list`未执行部署，是否继续执行.\033[0m +" +read -p "y/Y 继续执行安装,n/N 返回选择菜单：" yn + case $yn in + [Yy]*) + i=0 + #根据组合进行顺序安装 + list=`cat $NUM_DIR/common_list` + ins_names=(${list//,/ }) + oldnums=${#ins_names[*]} + for element in ${ins_names[@]} + do + ansible-playbook -i ../inventories/pro/hosts $element.yml + if [ $? -eq '0' ];then + sleep 10 + echoSuccess $element + recordinstnum $element + unset ins_names[$i] + let i++ + if [ $i -eq $oldnums ];then + cat /dev/null > $NUM_DIR/common_list + else + echo ${ins_names[@]} > $NUM_DIR/common_list + fi + else + echoError $element + exit 1 + fi + done + + break + ;; + [Nn]*) + cat /dev/null > $NUM_DIR/common_list + break;; + * ) + echo "Please answer yes or no." + ;; + esac +done + +} + +#组合安装调用方法 +function installcombination() { + +if [ -f $NUM_DIR/common_list ];then + if [ `cat $NUM_DIR/common_list | egrep "Mariadb|Nacos|Zookeeper|galaxy-gateway-keepalive|Kafka|Storm|Hadoop|HBase|Clickhouse|Druid|Spark|Arangodb" | wc -l` -gt '0' ] + then + abnormalCombination + else + installAllCommon + fi +else + installAllCommon +fi +clear +} +#=========================================commons======================================# + +#=========================================apps======================================# +#单个安装执行方法，根据传入的组件名称进行安装 +#若安装出现异常导致ansible停止，则脚本直接终止 +#若本机多次安装过某组件，则会提示是否继续安装；y 继续 N 停止 +function installationApps() { +NAME=$1 + +is=`getrecordnum $NAME` +if [ $is -lt '1' ];then + ansible-playbook -i ../inventories/pro/hosts $NAME.yml + if [ $? -eq '0' ];then + sleep 10 + echoSuccess $NAME + recordinstnum $NAME + else + echoError $NAME + exit 1 + fi +else + clear && echo -e "\033[31;1m本执行机已成功安装过 $is 次$NAME，缺是否继续安装，误操作可根据提示返回 \033[0m" && sleep 1 + while true; do + read -p "y/Y 继续执行安装,n/N 返回选择菜单：" yn + case $yn in + [Yy]*) + ansible-playbook -i ../inventories/pro/hosts $NAME.yml + if [ $? -eq '0' ];then + sleep 10 + echoSuccess $NAME + recordinstnum $NAME + break + else + echoError $NAME + exit 1 + fi + ;; + [Nn]*) + break;; + * ) + echo "Please answer yes or no." + ;; + esac +done +fi + +} + +#组合安装正常逻辑，逐个组件询问，Y执行 N不执行安装。 +#记录数组通过循环顺序执行安装 +function installAllApps() { +clear + +i=0 +hoskeep=`cat ../inventories/pro/config.yml | grep "hos_keepalive_need: yes" | wc -l` +if [[ $hoskeep -eq "1" ]]; then + ins_names=(galaxy-gateway-nginx galaxy-qgw-service galaxy-job-service galaxy-report-service galaxy-hos-service galaxy-chproxy galaxy-hos-nginx galaxy-hos-keepalive galaxy-gohangout) +else + ins_names=(galaxy-gateway-nginx galaxy-qgw-service galaxy-job-service galaxy-report-service galaxy-hos-service galaxy-chproxy galaxy-gohangout) +fi +echo ${ins_names[@]} > $NUM_DIR/app_list + +#根据组合进行顺序安装 +for element in ${ins_names[@]} +do + ansible-playbook -i ../inventories/pro/hosts $element.yml + if [ $? -eq '0' ];then + sleep 10 + echoSuccess $element + recordinstnum $element + unset ins_names[$i] + let i++ + echo ${ins_names[@]} > $NUM_DIR/app_list + else + echoError $element + exit 1 + fi +done +} + +#组合安装异常终止执行操作 +#询问是否需要再上次基础上执行，否则清空文件跳出循环 +function abnormalApps() { +clear +while true; do +echo -e " +\033[31;1m上次选择部署后还有 `cat $NUM_DIR/app_list`未执行部署，是否继续执行.\033[0m +" +read -p "y/Y 继续执行安装,n/N 返回选择菜单：" yn + case $yn in + [Yy]*) + i=0 + #根据组合进行顺序安装 + list=`cat $NUM_DIR/app_list` + ins_names=(${list//,/ }) + oldnums=${#ins_names[*]} + for element in ${ins_names[@]} + do + ansible-playbook -i ../inventories/pro/hosts $element.yml + if [ $? -eq '0' ];then + sleep 10 + echoSuccess $element + recordinstnum $element + unset ins_names[$i] + let i++ + if [ $i -eq $oldnums ];then + cat /dev/null > $NUM_DIR/app_list + else + echo ${ins_names[@]} > $NUM_DIR/app_list + fi + else + echoError $element + exit 1 + fi + done + break;; + [Nn]*) + cat /dev/null > $NUM_DIR/app_list + break;; + * ) + echo "Please answer yes or no." + ;; + esac +done + +} + +#组合安装调用方法 +function installApps() { + +if [ -f $NUM_DIR/app_list ];then + if [ `cat $NUM_DIR/app_list | egrep "galaxy-qgw-service|galaxy-job-service|galaxy-report-service|galaxy-hos-service|galaxy-chproxy|galaxy-gateway-nginx|galaxy-hos-nginx|galaxy-hos-keepalive|galaxy-gohangout" | wc -l` -gt '0' ] + then + abnormalApps + else + installAllApps + fi +else + installAllApps +fi +clear +} + +#======================================apps======================================# + +#======================================init======================================# +#组合安装正常逻辑，逐个组件询问，Y执行 N不执行安装。 +#记录数组通过循环顺序执行安装 +function initAll() { +clear + +i=0 +ins_names=(init-galaxy-gateway-keepalive init-kafka init-hbase init-druid init-clickhouse init-storm init-spark init-galaxy-hos-service) +echo ${ins_names[@]} > $NUM_DIR/init_list + +#根据组合进行顺序安装 +for element in ${ins_names[@]} +do + ansible-playbook -i ../inventories/pro/hosts $element.yml + if [ $? -eq '0' ];then + sleep 10 + echoSuccess $element + unset ins_names[$i] + let i++ + echo ${ins_names[@]} > $NUM_DIR/init_list + else + echoError $element + exit 1 + fi +done + +break +} + +#组合安装异常终止执行操作 +#询问是否需要再上次基础上执行，否则清空文件跳出循环 +function abnormalInit() { +clear +while true; do +echo -e " +\033[31;1m上次选择部署后还有 `cat $NUM_DIR/init_list`未执行部署，是否继续执行.\033[0m +" +read -p "y/Y 继续执行安装,n/N 返回选择菜单：" yn + case $yn in + [Yy]*) + i=0 + #根据组合进行顺序安装 + list=`cat $NUM_DIR/init_list` + ins_names=(${list//,/ }) + oldnums=${#ins_names[*]} + for element in ${ins_names[@]} + do + ansible-playbook -i ../inventories/pro/hosts $element.yml + if [ $? -eq '0' ];then + sleep 10 + echoSuccess $element + recordinstnum $element + unset ins_names[$i] + let i++ + if [ $i -eq $oldnums ];then + cat /dev/null > $NUM_DIR/init_list + else + echo ${ins_names[@]} > $NUM_DIR/init_list + fi + else + echoError $element + exit 1 + fi + done + break;; + [Nn]*) + cat /dev/null > $NUM_DIR/init_list + break;; + * ) + echo "Please answer yes or no." + ;; + esac +done + +} + +#组合安装调用方法 +function installInit() { + +if [ -f $NUM_DIR/init_list ];then + if [ `cat $NUM_DIR/init_list | egrep "init-galaxy-gateway-keepalive|init-kafka|init-hbase|init-druid|init-clickhouse|init-storm|init-spark|init-galaxy-job-service|init-galaxy-hos-service" | wc -l` -gt '0' ] + then + abnormalInit + else + initAll + fi +else + initAll +fi + +} +#======================================init======================================# + +#======================================check======================================# +#组合安装正常逻辑，逐个组件询问，Y执行 N不执行安装。 +#记录数组通过循环顺序执行安装 +function checkAll() { +clear + +i=0 +ins_names=(check-services check-components) +echo ${ins_names[@]} > $NUM_DIR/check_list + +#根据组合进行顺序安装 +for element in ${ins_names[@]} +do + ansible-playbook -i ../inventories/pro/hosts $element.yml + if [ $? -eq '0' ];then + echoSuccess $element + unset ins_names[$i] + let i++ + echo ${ins_names[@]} > $NUM_DIR/check_list + else + echoError $element + exit 1 + fi +done +clear +echo -e " +\033[32;1mCheck finished, service and component status is normal.\033[0m +" +sleep 5 +break +} + +#组合安装异常终止执行操作 +#询问是否需要再上次基础上执行，否则清空文件跳出循环 +function abnormalCheck() { +clear +while true; do +echo -e " +\033[31;1m上次选择部署后还有 `cat $NUM_DIR/check_list`未执行部署，是否继续执行.\033[0m +" +read -p "y/Y 继续执行安装,n/N 返回选择菜单：" yn + case $yn in + [Yy]*) + i=0 + #根据组合进行顺序安装 + list=`cat $NUM_DIR/check_list` + ins_names=(${list//,/ }) + oldnums=${#ins_names[*]} + for element in ${ins_names[@]} + do + ansible-playbook -i ../inventories/pro/hosts $element.yml + if [ $? -eq '0' ];then + sleep 10 + echoSuccess $element + recordinstnum $element + unset ins_names[$i] + let i++ + if [ $i -eq $oldnums ];then + cat /dev/null > $NUM_DIR/check_list + else + echo ${ins_names[@]} > $NUM_DIR/check_list + fi + else + echoError $element + exit 1 + fi + done + clear + echo -e " + \033[32;1mCheck finished, service and component status is normal.\033[0m + " + sleep 5 + break;; + [Nn]*) +# cat /dev/null > $NUM_DIR/check_list + break;; + * ) + echo "Please answer yes or no." + ;; + esac +done + +} + +#组合安装调用方法 +function installCheck() { + +if [ -f $NUM_DIR/check_list ];then + if [ `cat $NUM_DIR/check_list | egrep "check-services|check-components" | wc -l` -gt '0' ] + then + abnormalCheck + else + checkAll + fi +else + checkAll +fi + +} +#======================================check======================================# + + +while true; do + +if [[ -f $NUM_DIR/checkManager ]]; then + if [[ `cat $NUM_DIR/checkManager` -eq 0 ]]; then + checkManager + fi +else + checkManager +fi + +if [[ -f $NUM_DIR/checkCluster ]]; then + if [[ `cat $NUM_DIR/checkCluster` -eq 0 ]]; then + checkCluster + fi +else + checkCluster +fi + +clear +cat ../parcels/menu/homePage + +read -p " + Selection(1-4)? " yn +case $yn in + [1]) +while true; do +clear +#installcombination + +cat ../parcels/menu/commonMenu +echo " ********************************************************************************************************* " +echo " * * + * Press Ctrl+C or N to exit, Enter or Y to continue. * + * * + ********************************************************************************************************* + " + +read -p "Enter [Y] or [N]： " yn + case $yn in + [Yy]* ) + installcombination + break;; + [Nn]*) + break;; + * ) + echo "Please Enter [Y] or [N].";; + esac +done;; + [2] ) +while true; do +clear +#installApps + +cat ../parcels/menu/appMenu +read -p "Enter [Y] or [N]： " yn + case $yn in + [Yy]* ) + installApps + break;; + [Nn]* ) + break;; + * ) + echo "Please Enter [Y] or [N].";; + esac +done;; + [3] ) + installInit + ;; + [4] ) + installCheck + ;; +* ) + echo "Please Enter (1-4)." +;; +esac +done diff --git a/inventories/pro/group_vars/all b/inventories/pro/group_vars/all new file mode 100644 index 0000000..7a80fa4 --- /dev/null +++ b/inventories/pro/group_vars/all @@ -0,0 +1,617 @@ +####################### +#### #### +#### 基础配置 #### +#### #### +####################### +#============================基础配置===============================# +#根据对应的地区修改;默认为东八区，若为东六区，则修改为:"+06:00" +local_time: "+08:00" + +#若使用本安装包安装jdk，则此配置无需更改；若使用其他安装包或已安装过，则指向已有的jdk安装目录 +java_home: /usr/lib/jvm/jdk1.8.0_73 + +#统一安装路径 +install_path: /home/ceiec + +#数据存储路径，包含数据和日志 +data_path: /home/ceiec + +#若安装过程中检测到对应组件已存在，是否允许脚本卸载已存在的组件。 +#仅限本安装包部署的组件。 +allowed_unload: "yes" + +#============================bifang-Mariadb===============================# + +#Bifang MariaDB 地址，用以galaxy-qgw-service、galaxy-report-service的业务使用。 +bifang_mariadb_host: 127.0.0.1 + +#Bifang mariadb 数据库名 +bifang_mariadb_database: tsg-bifang + +#Bifang mariadb pin +bifang_mariadb_pin: 111111 + + +#galaxy mariadb key +galaxy_mariadb_pin: ceiec2019 + + +# 所有安装包路径,可自己指定绝对路径 +package_path: "{{ playbook_dir | dirname }}/software_packages" +#package_path: /opt/software_packages + + + +#============================虚IP===============================# + +#-------------apps------------------# + +#Galaxy 业务组件keepalive IP地址,填写的IP只能是和hosts-services配置的IP是同段的，且不能跟同段内已有的IP相同。 +gateway_keepalive_host: 192.168.44.23 + +#Galaxy 业务组件keepalive绑定网卡名称，不支持两个服务器网卡名称不同的情况，此情况需要安装完成后手动修改配置文件 +gateway_keepalive_interface: eth0 + +#Galaxy 业务组件keepalive router_id 注意同网段内此值不可重复 +gateway_keepalive_router_id: 61 + +#-------------hos------------------# +#是否需要对hos配置keepalive，若不需要则此处为no，若需要则为yes +#此项配置用于在服务器较少环境下，不对galaxy-hos-service服务单独做负载均衡的情况。 +hos_keepalive_need: "yes" + +#galaxy-hos-service服务keepalive IP地址 若 hos_keepalive_need为no，则此处填写gateway_keepalive_host对应的IP +#填写的IP只能是和hosts-hoskeepalive配置的IP是同段的，且不能跟同段内已有的IP相同。 +hos_keepalive_host: 127.0.0.2 + +#galaxy-hos-service服务keepalive绑定网卡名称，不支持两个服务器网卡名称不同的情况，此情况需要安装完成后手>动修改配置文件 +#若 hos_keepalive_need 配置no则此处 可不进行修改 +hos_keepalive_interface: eth0 + +#galaxy-hos-service服务keepalive router_id 注意同网段内此值不可重复 +#若 hos_keepalive_need 配置no则此处 可不进行修改 +hos_keepalive_router_id: 62 + + +#galaxy-keepalive的相关路径(目前gateway和hos共用这个配置) +keepalived_package_path: /home/keepalive + +keepalived_soft_home_path: galaxy/workspace/ + + +#============================Kafka===============================# +#任务消费kafka地址,若无特殊情况为本安装集群kafka地址，格式:kafkaip1:9092,kafkaip2:9092,kafkaip3:9092..... +consumer_kafka_servers: 127.0.0.1:9092 + +#任务结果写入kafka地址,若无特殊情况为本安装集群kafka地址，格式:kafkaip1:9092,kafkaip2:9092,kafkaip3:9092..... +producer_kafka_servers: 127.0.0.1:9092 + +#============================默认配置===============================# +#自研服务组件使用nacos 命名空间名称 +services_config_namespace: prod + +#Galaxy的 Nacos pin +nacos_pin: nacos + +#Nacos 使用的mariadb 数据库名称 +mariadb_nacos_database: nacos + +#Druid 使用的mariadb 数据库名称 +mariadb_druid_database: druid + +#galaxy-job-service依赖的MariaDB数据库名称 +mariadb_job_database: xxl_job + + + + +######################## +##### #### +##### 组件配置 #### +##### #### +######################## +#===========================druid===============================# +#druid服务使用磁盘最大大小，若druid单独使用服务器则配置，服务器可用空间的 75%，单位 字节 +#原配置名称：druid.server.maxSize +server_disk_maxsize: 5000000000000 + +#druid 本地最大的缓存大小 单位：字节 +#原配置名称：druid.segmentCache.locations +segmentCache_max_size: 300000000000 + +#========Druid-historical节点配置信息======== +#historical 内存大小，单位：MB +historical_mem: 512 + +#history 堆外内存指定大小，单位：MB +historical_MaxDirectMemorySize: 512 + +#historical buffer 大小 +#原配置名称：druid.processing.buffer.sizeBytes +historical_buffer_sizeBytes: 50000000 + +#historical numMergeBuffers 大小 +#原配置名称：druid.processing.numMergeBuffers +historical_numMergeBuffers: 4 + +#historical numThreads 大小 +#原配置名称：druid.processing.numThreads +historical_numThreads: 5 + +#========Druid-middlemanager节点配置信息======== +#middleManager 任务内存配置 +#原配置名称：druid.indexer.runner.javaOpts +middlemanager_runner_javaOpts: '-Xms512m -Xmx512m -XX:MaxDirectMemorySize=512m' + +#middleManager numMergeBuffers 大小 +#原配置名称：druid.indexer.fork.property.druid.processing.numMergeBuffers +middlemanager_numMergeBuffers: 2 + +#middleManager buffer.sizeBytes 大小 +#原配置名称：druid.indexer.fork.property.druid.processing.buffer.sizeBytes +middlemanager_buffer_sizeBytes: 20000000 + +#middleManager numThreads 大小 +#原配置名称：druid.indexer.fork.property.druid.processing.numThreads +middlemanager_numThreads: 1 + +#========Druid-coordinator节点配置信息======== +#coordinator-overlord 内存大小，单位：MB +coordinator_mem: 512 + +#========Druid-broker节点配置信息======== +#broker 内存大小，单位：MB +broker_mem: 512 + +#broker 堆外内存指定大小，单位：MB +broker_MaxDirectMemorySize: 1024 + +#druid处理缓存大小配置，单位：字节 +#原配置名称：druid.processing.buffer.sizeBytes +broker_sizeBytes: 50000000 + +#druid处理mergeBuffer大小配置 +#原配置名称：druid.processing.numMergeBuffers +broker_numMergeBuffers: 6 + +#druid处理线程数量配置 +#原配置名称：druid.processing.numThreads +broker_numThreads: 1 + +#===========================Hadoop===============================# +#namenode 进程使用内存大小，单位：MB +namenode_mem: 512 + +#datanode 进程使用内存大小，单位：MB +datanode_mem: 512 + +#journalnode 进程使用内存大小，单位：MB +journal_mem: 1024 + +#zkfc 进程使用内存大小，单位：MB +zkfc_mem: 1024 + +#namenode 工作线程池大小 +#原配置名称：dfs.namenode.handler.count +namenode_handlers: 40 + +#datanode 工作线程池大小 +#原配置名称：dfs.datanode.handler.count +datanode_handlers: 40 + +#===========================HBase===============================# +#Hmaster进程使用的内存，单位：MB +master_mem: 1024 + +#Hregionserver进程使用的内存，单位：MB +region_mem: 2048 + +#Hregionserver进程young内存大小，单位：MB +region_young_mem: 512 + +#regionserver处理线程数，一般和cpu数相同，也可为核数的整数倍 +#原配置名称：hbase.regionserver.handler.count +regionserverhandlers: 32 + +#===========================Kafka===============================# +#kafka 进程使用最大内存，单位：MB +kafka_max_mem: 1024 + +#kafka 进程初始最小内存，单位：MB +kafka_min_mem: 512 + +#kafka数据存储时间 单位：小时，默认 168 +log_reten_hours: 168 + +#topic一个分区最大存储多少字节，默认 1G +topic_max_bytes: 10737418240 + +#topic一个分区内单个数据文件最大多少字节，默认 1G +#超过会删除，但删除需要时间，即一个partition下可能存在新旧多个文件，一般旧文件不超过两个。 +topic_file_bytes: 1073741824 + +#===========================Mariadb===============================# +#缓冲池字节大小 单位，单位：MB +mariadb_innodb_buffer_pool_size: 1024 + +#===========================Spark===============================# +#spark worker 节点内存，单位：MB +spark_worker_mem: 512 + +#spark worker节点处理线程数 +spark_worker_cores: 30 + +#===========================Nacos===============================# +#Nacos 内存配置 +nacos_java_opt: '-Xms1024m -Xmx1024m -Xmn1024m' + +#===========================Storm===============================# +#单个supervisor可使用worker数量，一般为CPU的一半。 +storm_slot_nums: 25 + +#单个worker内存大小，单位：MB +storm_worker_max_mem: 1024 + +#单个worker内存大小，单位：MB +storm_worker_min_mem: 1024 + +#===========================Zookeeper===============================# +#zookeeper 进程 启动内存大小 单位MB +zookeeper_max_mem: 1024 + +#===========================Mariadb===============================# +#mariadb 端口 +galaxy_mariadb_port: 3306 + +#mariadb 用户 +galaxy_mariadb_username: root + + + +######################## +##### #### +##### 业务配置 #### +##### #### +######################## +#=======================galaxy-hos-service==========================# +#galaxy-hos-service服务 tomcat缓存大小,单位KB +hos_tomcat_cache_size: 100000 + +#galaxy-hos-service 服务内存设置 +hos_java_opts: "-Xmx512m -Xms512m -Xmn512m" + +#=======================galaxy-qgw-service==========================# +#galaxy-qgw-service 服务内存设置 +qgw_java_opts: "-Xmx512m -Xms512m" + +#=======================galaxy-job-service==========================# +#galaxy-job-service 服务内存设置 +job_java_opts: "-Xmx512m -Xms512m" + +#======================galaxy-report-service=========================# +#galaxy-report-service 服务内存设置 +report_java_opts: "-Xmx512m -Xms512m" + + + + + + + + +######################## +##### #### +##### 开发者配置 #### +##### #### +######################## + +#------zookeeper---------# +zookeeper_version: zookeeper-3.4.10 + +#zookeeper myid起始id，如不是扩容则此处无需修改；若为扩容，则此处改为原集群节点 myid最大值+1 +zk_start_myid: 1 + +#zookeeper port +zookeeper_port: 2181 + + +#------mariadb---------# +#mariadb数据目录 +mariadb_datadir_path: /data/mariadb + + + +#-----nacos---------# +#nacos version +nacos_version: nacos-2.0.2 + +#nacos local port +nacos_port: 8847 + + + +#-----kafka---------# +#Kafka版本 +kafka_version: kafka_2.11-1.0.0 + +#kafka-manager 版本 +kafka_manager_version: kafka-manager-1.3.3.18 + +#kafka broker起始id +kafka_start_broker: 1 + +#kafka default port +server_port: 9092 + + + +#------storm---------# +storm_version: apache-storm-1.0.2 + + + +#-------hadoop---------# +hadoop_version: hadoop-2.7.1 + + + +#--------Hbase---------# +hbase_version: hbase-2.2.3 + + +#------Clickhouse---------# +#clickhouse 版本 +clickhouse_version: 20.3.12.112 + +#clickhouse默认的数据库 +default_database: default + +#clickhouse 只读key +readonly_key: ceiec2018 + +#clickhouse 管理key +root_key: ceiec2019 + + + +#-------druid---------# +#druid 版本 +druid_version: apache-druid-0.18.1 + +#druid 集群数量限制 +cluster_limit: "4" + + + +#------Spark---------# +spark_version: spark-2.2.3-bin-hadoop2.7 + +scala_version: scala-2.11.7 + +#请勿修改此配置，安装过程会自动安装此版本scala。 +scala_home: /usr/lib/scala/scala-2.11.7 + + + + +#-----arangodb---------# +#注意这里面定义的key中不能包含减号-,否则程序无法识别定义的key +arangodb_password: ceiec2018 + +#离线模式下使用:bifang-api的docker镜像文件名称,在../file中的文件名称 +arangodb_image_tar_name: arangodb-3.6.4.tar + +##########################公共属性############################# +#设置容器的hostname +#container_hostname: bifang-api +#{{ image_tar_name }}中的镜像名称,或者{{ docker_registry_image_and_tag }}中的镜像名称,不含版本号 +arangodb_image_name: arangodb +#{{ image_tar_name }}的镜像版本号,或者{{ docker_registry_image_and_tag }}中的镜像版本号 +arangodb_image_tag_name: 3.6.4 +#{{ image_tar_name }}的镜像,或者{{ docker_registry_image_and_tag }}中的镜像启动后的容器名称 +arangodb_image_container_name: arangodb + +#镜像文件,docker-compose文件以及数据卷共享目录所在的位置,注意路径最后面不带/ +arangodb_soft_home_path: galaxy/workspace/arangodb + +#挂载目录的相对路径，不要随意修改 +arango_volume_path: galaxy/volumes/arangodb + + + + +#------environment---------# +#注意这里面定义的key中不能包含减号-,否则程序无法识别定义的key +jdk_home_path: /usr/lib/jvm + +#注意这里面定义的key中不能包含减号-,否则程序无法识别定义的key +#执行脚本的目录 +test_soft_home_path: /home/test/check_begin + + + +#--------galaxy-chproxy---------# +#注意这里面定义的key中不能包含减号-,否则程序无法识别定义的key +#离线模式下使用:bifang-api的docker镜像文件名称,在../file中的文件名称 +chproxy_image_tar_name: chproxy-21.06.30.tar + +##########################公共属性############################# +#{{ image_tar_name }}中的镜像名称,或者{{ docker_registry_image_and_tag }}中的镜像名称,不含版本号 +chproxy_image_name: galaxy-chproxy + +#{{ image_tar_name }}的镜像版本号,或者{{ docker_registry_image_and_tag }}中的镜像版本号 +chproxy_image_tag_name: 21.06.30 + +#{{ image_tar_name }}的镜像,或者{{ docker_registry_image_and_tag }}中的镜像启动后的容器名称 +chproxy_image_container_name: galaxy-chproxy + +#镜像文件,docker-compose文件以及数据卷共享目录所在的位置,注意路径最后面不带/ +chproxy_soft_home_path: galaxy/workspace/galaxy-chproxy + +#挂载目录的相对路径，不要随意修改 +chproxy_volume_path: galaxy/volumes/galaxy-chproxy + + + + + +#-------galaxy-gateway-nginx---------# +gateway_image_tar_name: nginx-metrics-1.17.0.tar + +#{{ image_tar_name }}中的镜像名称,或者{{ docker_registry_image_and_tag }}中的镜像名称,不含版本号 +gateway_image_name: galaxy-gateway-nginx + +##{{ image_tar_name }}的镜像版本号,或者{{ docker_registry_image_and_tag }}中的镜像版本号 +gateway_image_tag_name: 1.17.0 + +##{{ image_tar_name }}的镜像,或者{{ docker_registry_image_and_tag }}中的镜像启动后的容器名称 +gateway_image_container_name: galaxy-gateway-nginx + +#宿主机存放与nginx容器映射的data目录和log的文件夹 +gateway_nginx_soft_home_path: galaxy/workspace/galaxy-gateway-nginx #注意路径最后面没有/ + +#nginx 数据目录 +gateway_nginx_volume_path: galaxy/volumes/galaxy-gateway-nginx + + + + +#--------galaxy-gohangout---------# +#注意这里面定义的key中不能包含减号-,否则程序无法识别定义的key + +#离线模式下使用:bifang-api的docker镜像文件名称,在../file中的文件名称 +gohangout_image_tar_name: gohangout-1.15.2.20210408.tar + +##########################公共属性############################# +#{{ image_tar_name }}中的镜像名称,或者{{ docker_registry_image_and_tag }}中的镜像名称,不含版本号 +gohangout_image_name: gohangout +#{{ image_tar_name }}的镜像版本号,或者{{ docker_registry_image_and_tag }}中的镜像版本号 +gohangout_image_tag_name: 1.15.2.20210408 +#{{ image_tar_name }}的镜像,或者{{ docker_registry_image_and_tag }}中的镜像启动后的容器名称 +gohangout_image_container_name: gohangout +#镜像文件,docker-compose文件以及数据卷共享目录所在的位置,注意路径最后面不带/ +gohangout_soft_home_path: galaxy/workspace/gohangout + +#挂载容器目录的相对路径，不要随意更改 +gohangout_volume_path: galaxy/volumes/gohangout + + + +#--------galaxy-hos-nginx---------# +hos_nginx_image_tar_name: nginx-metrics-1.17.0.tar + +#{{ image_tar_name }}中的镜像名称,或者{{ docker_registry_image_and_tag }}中的镜像名称,不含版本号 +hos_nginx_image_name: galaxy-hos-nginx + +##{{ image_tar_name }}的镜像版本号,或者{{ docker_registry_image_and_tag }}中的镜像版本号 +hos_nginx_image_tag_name: 1.17.0 + +##{{ image_tar_name }}的镜像,或者{{ docker_registry_image_and_tag }}中的镜像启动后的容器名称 +hos_nginx_image_container_name: galaxy-hos-nginx + +#宿主机存放与nginx容器映射的data目录和log的文件夹 +hos_nginx_soft_home_path: galaxy/workspace/galaxy-hos-nginx #注意路径最后面没有/ + +#nginx 数据目录 +hos_nginx_volume_path: galaxy/volumes/galaxy-hos-nginx + + + +#---------galaxy-hos-service---------# +#离线模式下使用:bifang-api的docker镜像文件名称,在../file中的文件名称 +hos_service_image_tar_name: galaxy-hos-service-21.07.01.tar + +##########################公共属性############################# +#{{ image_tar_name }}中的镜像名称,或者{{ docker_registry_image_and_tag }}中的镜像名称,不含版本号 +hos_service_image_name: galaxy-hos-service + +#{{ image_tar_name }}的镜像版本号,或者{{ docker_registry_image_and_tag }}中的镜像版本号 +hos_service_image_tag_name: 21.07.01 + +#{{ image_tar_name }}的镜像,或者{{ docker_registry_image_and_tag }}中的镜像启动后的容器名称 +hos_service_image_container_name: galaxy-hos-service + +#镜像文件,docker-compose文件以及数据卷共享目录所在的位置,注意路径最后面不带/ +hos_service_soft_home_path: galaxy/workspace/galaxy-hos-service + +#挂载目录的相对路径，不要随意修改 +hos_volume_path: galaxy/volumes/galaxy-hos-service + + + +#---------galaxy-job-service.yml---------# +#离线模式下使用:bifang-api的docker镜像文件名称,在../file中的文件名称 +job_image_tar_name: galaxy-job-executor-v1.3.210413-rc1.tar +admin_image_tar_name: galaxy-job-admin-v1.3.20210408.tar + +##########################公共属性############################# +#设置容器的hostname +#{{ image_tar_name }}中的镜像名称,或者{{ docker_registry_image_and_tag }}中的镜像名称,不含版本号 +job_image_name: galaxy-job-executor +admin_image_name: galaxy-job-admin + +#{{ image_tar_name }}的镜像版本号,或者{{ docker_registry_image_and_tag }}中的镜像版本号 +job_image_tag_name: v1.3.210413-rc1 +admin_image_tag_name: v1.3.20210408 + +#{{ image_tar_name }}的镜像,或者{{ docker_registry_image_and_tag }}中的镜像启动后的容器名称 +job_image_container_name: galaxy-job-executor +admin_image_container_name: galaxy-job-admin + +#镜像文件,docker-compose文件以及数据卷共享目录所在的位置,注意路径最后面不带/ +job_soft_home_path: galaxy/workspace/galaxy-job/galaxy-job-executor +admin_soft_home_path: galaxy/workspace/galaxy-job/galaxy-job-admin + +#挂载目录的相对路径，不要随意修改 +job_volume_path: galaxy/volumes/galaxy-job/galaxy-job-executor +admin_volume_path: galaxy/volumes/galaxy-job/galaxy-job-admin + + + + + +#---------galaxy-qgw-service---------# +#注意这里面定义的key中不能包含减号-,否则程序无法识别定义的key + +#离线模式下使用:bifang-api的docker镜像文件名称,在../file中的文件名称 +qgw_image_tar_name: galaxy-qgw-service-345.tar + +##########################公共属性############################# +#{{ image_tar_name }}中的镜像名称,或者{{ docker_registry_image_and_tag }}中的镜像名称,不含版本号 +qgw_image_name: galaxy-qgw-service + +#{{ image_tar_name }}的镜像版本号,或者{{ docker_registry_image_and_tag }}中的镜像版本号 +qgw_image_tag_name: 345 + +#{{ image_tar_name }}的镜像,或者{{ docker_registry_image_and_tag }}中的镜像启动后的容器名称 +qgw_image_container_name: galaxy-qgw-service + +#镜像文件,docker-compose文件以及数据卷共享目录所在的位置,注意路径最后面不带/ +qgw_soft_home_path: galaxy/workspace/galaxy-qgw-service + +#挂载目录的相对路径，不要随意修改 +qgw_volume_path: galaxy/volumes/galaxy-qgw-service + + + + +#---------galaxy-report-service---------# +#注意这里面定义的key中不能包含减号-,否则程序无法识别定义的key +#离线模式下使用:bifang-api的docker镜像文件名称,在../file中的文件名称 +report_image_tar_name: galaxy-report-service-21.04.07.tar + +##########################公共属性############################# +#{{ image_tar_name }}中的镜像名称,或者{{ docker_registry_image_and_tag }}中的镜像名称,不含版本号 +report_image_name: galaxy-report-service + +#{{ image_tar_name }}的镜像版本号,或者{{ docker_registry_image_and_tag }}中的镜像版本号 +report_image_tag_name: 21.04.07 + +#{{ image_tar_name }}的镜像,或者{{ docker_registry_image_and_tag }}中的镜像启动后的容器名称 +report_image_container_name: galaxy-report-service + +#镜像文件,docker-compose文件以及数据卷共享目录所在的位置,注意路径最后面不带/ +report_soft_home_path: galaxy/workspace/galaxy-report-service + +#report 数据目录 +report_volume_path: galaxy/volumes/galaxy-report-service diff --git a/inventories/pro/hosts b/inventories/pro/hosts new file mode 100644 index 0000000..617f06e --- /dev/null +++ b/inventories/pro/hosts @@ -0,0 +1,49 @@ +#此标签指定的IP用于：chproxy、galaxy-app-nginx、galaxy-job-service、galaxy-qgw-service、galaxy-report-service、galaxy-app-keepalive的安装 +#最少需要指定两台。 +[services] + +#此标签指定的IP用于：galaxy-hos-service 的安装 +[hos] + +#此标签指定的IP用于：galaxy-hos-keepalive和galaxy-hos-nginx的安装 +#config.yml内的hos_keepalive_need配置是no，则此处不写任何IP，yes就写部署galaxy-hos-service服务keepalive的服务器地址。 +#不能与services指定的IP相同,且最多为两台。 +[hoskeepalive] + +#此标签指定的IP用于：ArangoDB的安装 +[arangodb] + +#此标签指定的IP用于：Mariadb的安装，目前版本下仅可配置两个IP，且需要与services标签的前两个IP相同。 +#现没有负载需求，保留后期做负载的功能需求。 +[mariadb] + +#此标签指定的IP用于：Nacos的安装 +[nacos] + +#此标签指定的IP用于：Apache Zookeeper的安装 +[zookeeper] + +#此标签指定的IP用于：Apache Kafka的安装，集群第一台同时部署Kafka-Manager服务。 +[kafka] + +#此标签指定的IP用于：Apache Storm的安装，根据内部约定，若集群整体小于3台则第一台为numbus节点；大于三台则前三台为nimbus节点。 +[storm] + +#此标签指定的IP用于：Apache Hadoop的安装 +[hadoop] + +#此标签指定的IP用于：Apache HBase的安装 +[hbase] + +#此标签指定的IP用于：Apache Spark的安装，根据内部约定，若集群小于3台则worker在所有节点上；大于3台则第一个节点为master节点。 +[spark] + +#此标签指定的IP用于：Galaxy-Gohangout的安装 +[gohangout] + +#此标签指定的IP用于：Apache Druid的安装，根据内部约定，若集群整体小于4台则每台为全部节点；大于4台则前两台默认为query节点。 +[druid] + +#此标签指定的IP用于：Clickhouse的安装,根据内部约定，前两台默认为query节点。 +[clickhouse] + diff --git a/parcels/Arangodb.yml b/parcels/Arangodb.yml new file mode 100644 index 0000000..4358560 --- /dev/null +++ b/parcels/Arangodb.yml @@ -0,0 +1,8 @@ +- hosts: arangodb + remote_user: root + roles: + - components/arangodb +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - roles/components/arangodb/vars/main.yml diff --git a/parcels/Clickhouse.yml b/parcels/Clickhouse.yml new file mode 100644 index 0000000..4e5c75c --- /dev/null +++ b/parcels/Clickhouse.yml @@ -0,0 +1,28 @@ + +- hosts: clickhouse + remote_user: root + roles: + - components/clickhouse/common +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - roles/components/clickhouse/common/vars/main.yml + +- hosts: clickhouse + remote_user: root + roles: + - components/clickhouse/query +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - roles/components/clickhouse/common/vars/main.yml + +- hosts: clickhouse + remote_user: root + roles: + - components/clickhouse/data +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - roles/components/clickhouse/common/vars/main.yml + diff --git a/parcels/Druid.yml b/parcels/Druid.yml new file mode 100644 index 0000000..d2bd21a --- /dev/null +++ b/parcels/Druid.yml @@ -0,0 +1,8 @@ +- hosts: druid + remote_user: root + roles: + - components/druid +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - roles/components/druid/vars/main.yml diff --git a/parcels/HBase.yml b/parcels/HBase.yml new file mode 100644 index 0000000..248d4be --- /dev/null +++ b/parcels/HBase.yml @@ -0,0 +1,10 @@ +- hosts: hbase + remote_user: root + roles: + - components/hbase +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/components/hbase/vars/main.yml + diff --git a/parcels/Hadoop.yml b/parcels/Hadoop.yml new file mode 100644 index 0000000..ea24036 --- /dev/null +++ b/parcels/Hadoop.yml @@ -0,0 +1,11 @@ +- hosts: hadoop + remote_user: root + roles: + - components/hadoop +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/components/hadoop/vars/main.yml + + diff --git a/parcels/Kafka.yml b/parcels/Kafka.yml new file mode 100644 index 0000000..a67f0ee --- /dev/null +++ b/parcels/Kafka.yml @@ -0,0 +1,11 @@ + +- hosts: kafka + remote_user: root + roles: + - components/kafka +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/components/kafka/vars/main.yml + diff --git a/parcels/Mariadb.yml b/parcels/Mariadb.yml new file mode 100644 index 0000000..5e05084 --- /dev/null +++ b/parcels/Mariadb.yml @@ -0,0 +1,20 @@ + +- hosts: mariadb + remote_user: root + roles: + - components/mariadb/install-mariadb-cluster +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/components/mariadb/install-mariadb-cluster/vars/main.yml + +- hosts: mariadb + remote_user: root + roles: + - components/mariadb/mariadb-double-master +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/components/mariadb/install-mariadb-cluster/vars/main.yml diff --git a/parcels/Nacos.yml b/parcels/Nacos.yml new file mode 100644 index 0000000..6ebfd98 --- /dev/null +++ b/parcels/Nacos.yml @@ -0,0 +1,10 @@ + +- hosts: nacos + remote_user: root + roles: + - components/nacos +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/components/nacos/vars/main.yml diff --git a/parcels/Spark.yml b/parcels/Spark.yml new file mode 100644 index 0000000..2f6a874 --- /dev/null +++ b/parcels/Spark.yml @@ -0,0 +1,10 @@ + +- hosts: spark + remote_user: root + roles: + - components/spark +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/components/spark/vars/main.yml diff --git a/parcels/Storm.yml b/parcels/Storm.yml new file mode 100644 index 0000000..c06dcff --- /dev/null +++ b/parcels/Storm.yml @@ -0,0 +1,11 @@ + +- hosts: storm + remote_user: root + roles: + - components/storm +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/components/storm/vars/main.yml + diff --git a/parcels/Zookeeper.yml b/parcels/Zookeeper.yml new file mode 100644 index 0000000..5c454b3 --- /dev/null +++ b/parcels/Zookeeper.yml @@ -0,0 +1,12 @@ + +- hosts: zookeeper + remote_user: root +# max_fail_percentage: 0 + roles: + - components/zookeeper +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/components/zookeeper/vars/main.yml + diff --git a/parcels/check-components.yml b/parcels/check-components.yml new file mode 100644 index 0000000..9e97bd1 --- /dev/null +++ b/parcels/check-components.yml @@ -0,0 +1,6 @@ +- hosts: all + remote_user: root + roles: + - check/check-components +# vars_files: +# - ../configurations/config.yml diff --git a/parcels/check-services.yml b/parcels/check-services.yml new file mode 100644 index 0000000..b23a79b --- /dev/null +++ b/parcels/check-services.yml @@ -0,0 +1,6 @@ +- hosts: services + remote_user: root + roles: + - check/check-services +# vars_files: +# - ../configurations/config.yml diff --git a/parcels/environment.yml b/parcels/environment.yml new file mode 100644 index 0000000..8c8b1d1 --- /dev/null +++ b/parcels/environment.yml @@ -0,0 +1,25 @@ +#安装JDK +- hosts: all + remote_user: root + roles: + - dependencies/install-jdk +# vars_files: +# - roles/dependencies/install-jdk/vars/main.yml +# - ../configurations/config.yml + +#安装pip和docker +- hosts: all + remote_user: root + roles: + - dependencies/install-docker + - dependencies/install-pip +# vars_files: +# - ../configurations/config.yml + +- hosts: all + remote_user: root + roles: + - dependencies/check-env +# vars_files: +# - roles/dependencies/check-env/vars/main.yml +# - ../configurations/config.yml diff --git a/parcels/galaxy-chproxy.yml b/parcels/galaxy-chproxy.yml new file mode 100644 index 0000000..cacd5fa --- /dev/null +++ b/parcels/galaxy-chproxy.yml @@ -0,0 +1,9 @@ +- hosts: services + remote_user: root + roles: + - services/galaxy-chproxy +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/services/galaxy-chproxy/vars/main.yml diff --git a/parcels/galaxy-gateway-keepalive.yml b/parcels/galaxy-gateway-keepalive.yml new file mode 100644 index 0000000..aced81b --- /dev/null +++ b/parcels/galaxy-gateway-keepalive.yml @@ -0,0 +1,9 @@ +- hosts: services + remote_user: root + roles: + - services/galaxy-gateway-keepalive +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/services/galaxy-gateway-keepalive/vars/main.yml diff --git a/parcels/galaxy-gateway-nginx.yml b/parcels/galaxy-gateway-nginx.yml new file mode 100644 index 0000000..5ec6fd0 --- /dev/null +++ b/parcels/galaxy-gateway-nginx.yml @@ -0,0 +1,9 @@ +- hosts: services + remote_user: root + roles: + - services/galaxy-gateway-nginx +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/services/galaxy-gateway-nginx/vars/main.yml diff --git a/parcels/galaxy-gohangout.yml b/parcels/galaxy-gohangout.yml new file mode 100644 index 0000000..f23615b --- /dev/null +++ b/parcels/galaxy-gohangout.yml @@ -0,0 +1,9 @@ +- hosts: gohangout + remote_user: root + roles: + - services/galaxy-gohangout +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/services/galaxy-gohangout/vars/main.yml diff --git a/parcels/galaxy-hos-keepalive.yml b/parcels/galaxy-hos-keepalive.yml new file mode 100644 index 0000000..feda7d4 --- /dev/null +++ b/parcels/galaxy-hos-keepalive.yml @@ -0,0 +1,9 @@ +- hosts: hoskeepalive + remote_user: root + roles: + - services/galaxy-hos-keepalive +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/services/galaxy-hos-keepalive/vars/main.yml diff --git a/parcels/galaxy-hos-nginx.yml b/parcels/galaxy-hos-nginx.yml new file mode 100644 index 0000000..3520610 --- /dev/null +++ b/parcels/galaxy-hos-nginx.yml @@ -0,0 +1,9 @@ +- hosts: hoskeepalive + remote_user: root + roles: + - services/galaxy-hos-nginx +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/services/galaxy-hos-nginx/vars/main.yml diff --git a/parcels/galaxy-hos-service.yml b/parcels/galaxy-hos-service.yml new file mode 100644 index 0000000..94b8bab --- /dev/null +++ b/parcels/galaxy-hos-service.yml @@ -0,0 +1,10 @@ +- hosts: hos + remote_user: root + roles: + - services/galaxy-hos-service +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/services/galaxy-hos-service/vars/main.yml + diff --git a/parcels/galaxy-job-service.yml b/parcels/galaxy-job-service.yml new file mode 100644 index 0000000..6c2f018 --- /dev/null +++ b/parcels/galaxy-job-service.yml @@ -0,0 +1,9 @@ +- hosts: services + remote_user: root + roles: + - services/galaxy-job-service +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/services/galaxy-job-service/vars/main.yml diff --git a/parcels/galaxy-qgw-service.yml b/parcels/galaxy-qgw-service.yml new file mode 100644 index 0000000..300caca --- /dev/null +++ b/parcels/galaxy-qgw-service.yml @@ -0,0 +1,10 @@ +- hosts: services + remote_user: root + roles: + - services/galaxy-qgw-service +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/services/galaxy-qgw-service/vars/main.yml + diff --git a/parcels/galaxy-report-service.yml b/parcels/galaxy-report-service.yml new file mode 100644 index 0000000..89cb4ae --- /dev/null +++ b/parcels/galaxy-report-service.yml @@ -0,0 +1,10 @@ +- hosts: services + remote_user: root + roles: + - services/galaxy-report-service +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/services/galaxy-report-service/vars/main.yml + diff --git a/parcels/init-clickhouse.yml b/parcels/init-clickhouse.yml new file mode 100644 index 0000000..55538f3 --- /dev/null +++ b/parcels/init-clickhouse.yml @@ -0,0 +1,10 @@ +- hosts: + - clickhouse + remote_user: root + roles: + - init/clickhouse +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/components/clickhouse/common/vars/main.yml diff --git a/parcels/init-druid.yml b/parcels/init-druid.yml new file mode 100644 index 0000000..e31e6e2 --- /dev/null +++ b/parcels/init-druid.yml @@ -0,0 +1,9 @@ +- hosts: druid + remote_user: root + roles: + - init/druid +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/components/druid/vars/main.yml diff --git a/parcels/init-galaxy-gateway-keepalive.yml b/parcels/init-galaxy-gateway-keepalive.yml new file mode 100644 index 0000000..8e2f6f7 --- /dev/null +++ b/parcels/init-galaxy-gateway-keepalive.yml @@ -0,0 +1,9 @@ +- hosts: services + remote_user: root + roles: + - init/galaxy-gateway-keepalive +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/services/galaxy-gateway-keepalive/vars/main.yml diff --git a/parcels/init-galaxy-hos-service.yml b/parcels/init-galaxy-hos-service.yml new file mode 100644 index 0000000..e0fa1b9 --- /dev/null +++ b/parcels/init-galaxy-hos-service.yml @@ -0,0 +1,9 @@ +- hosts: services + remote_user: root + roles: + - init/galaxy-hos-service +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/services/galaxy-hos-service/vars/main.yml diff --git a/parcels/init-galaxy-job-service.yml b/parcels/init-galaxy-job-service.yml new file mode 100644 index 0000000..8ee4b82 --- /dev/null +++ b/parcels/init-galaxy-job-service.yml @@ -0,0 +1,10 @@ +- hosts: + - services + remote_user: root + roles: + - init/galaxy-job-service +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/services/galaxy-job-service/vars/main.yml diff --git a/parcels/init-hbase.yml b/parcels/init-hbase.yml new file mode 100644 index 0000000..3d2d9bc --- /dev/null +++ b/parcels/init-hbase.yml @@ -0,0 +1,10 @@ +- hosts: + - hbase + remote_user: root + roles: + - init/hbase +# vars_files: +# - roles/components/hbase/vars/main.yml +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml diff --git a/parcels/init-kafka.yml b/parcels/init-kafka.yml new file mode 100644 index 0000000..c13bc82 --- /dev/null +++ b/parcels/init-kafka.yml @@ -0,0 +1,10 @@ +- hosts: + - kafka + remote_user: root + roles: + - init/kafka +# vars_files: +# - roles/components/kafka/vars/main.yml +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml diff --git a/parcels/init-spark.yml b/parcels/init-spark.yml new file mode 100644 index 0000000..1ddba8a --- /dev/null +++ b/parcels/init-spark.yml @@ -0,0 +1,10 @@ +- hosts: + - spark + remote_user: root + roles: + - init/spark +# vars_files: +# - roles/components/spark/vars/main.yml +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml diff --git a/parcels/init-storm.yml b/parcels/init-storm.yml new file mode 100644 index 0000000..3b41866 --- /dev/null +++ b/parcels/init-storm.yml @@ -0,0 +1,10 @@ +- hosts: + - storm + remote_user: root + roles: + - init/storm +# vars_files: +# - roles/components/storm/vars/main.yml +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml diff --git a/parcels/menu/appMenu b/parcels/menu/appMenu new file mode 100644 index 0000000..4deace8 --- /dev/null +++ b/parcels/menu/appMenu @@ -0,0 +1,20 @@ +Service Components： + + ┌──────────────────────────────────────┐ + ├ galaxy-qgw-service ┤ + ├ galaxy-job-service ┤ + ├ galaxy-report-service ┤ + ├ galaxy-hos-service ┤ + ├ galaxy-chproxy ┤ + ├ galaxy-gohangout ┤ + ├ galaxy-gateway-nginx ┤ + ├ galaxy-nginx-hos ┤ + ├ galaxy-gateway-keepalive ┤ + ├ galaxy-hos-keepalive ┤ + └──────────────────────────────────────┘ + +********************************************************* +* Press Ctrl+C or N to exit, Enter or Y to continue. * +* * +********************************************************* + diff --git a/parcels/menu/commonMenu b/parcels/menu/commonMenu new file mode 100644 index 0000000..e688de4 --- /dev/null +++ b/parcels/menu/commonMenu @@ -0,0 +1,17 @@ +Open Source Software： + + ┌────────────────────────────────┐ + ├ [*] Apache Zookeeper ┤ + ├ [*] Mariadb ┤ + ├ [*] Nacos ┤ + ├ [*] Apache Kafka ┤ + ├ [*] Apache Storm ┤ + ├ [*] Apache Hadoop ┤ + ├ [*] Apache HBase ┤ + ├ [*] Clickhouse ┤ + ├ [*] Apache Druid ┤ + ├ [*] Apache Spark ┤ + ├ [*] ArangoDB ┤ + └────────────────────────────────┘ + + diff --git a/parcels/menu/homePage b/parcels/menu/homePage new file mode 100644 index 0000000..0873861 --- /dev/null +++ b/parcels/menu/homePage @@ -0,0 +1,15 @@ +Galaxy Data Platform Cluster - TSG21.06 + + Please choose one of the following: + +┌────────────────────────────────────────────────────┐ +├ Step 1：Install Open Source Software ┤ +├ Step 2：Install Service Components ┤ +├ Step 3：Install Init Script ┤ +├ Step 4：Test & Validation ┤ +└────────────────────────────────────────────────────┘ + +****************************************************** +* Press Ctrl+C (^C) to exit, Enter to continue. * +****************************************************** + diff --git a/parcels/roles/check/check-components/tasks/main.yml b/parcels/roles/check/check-components/tasks/main.yml new file mode 100644 index 0000000..1971dfa --- /dev/null +++ b/parcels/roles/check/check-components/tasks/main.yml @@ -0,0 +1,25 @@ +- name: Checking Strom Topology Num + shell: "source /etc/profile && storm list | grep 'ACTIVE' | wc -l" + register: stormlist + run_once: true + delegate_to: '{{ groups.storm[0] }}' + +- name: Checking Strom Topology Num + fail: + msg: "storm 提交任务数量与版本对应任务数量不匹配，请检查" + when: stormlist.stdout < '15' + run_once: true + delegate_to: '{{ groups.storm[0] }}' + +- name: Checking Gohangout Num + shell: "ps -ef | grep gohangout | grep worker | grep -v grep | wc -l" + register: golist + run_once: true + delegate_to: '{{ groups.gohangout[0] }}' + +- name: Checking Gohangout Num + fail: + msg: "Gohangout 任务启动数据量与版本对应任务数量不匹配，请检查" + when: golist.stdout < '10' + run_once: true + delegate_to: '{{ groups.gohangout[0] }}' diff --git a/parcels/roles/check/check-services/tasks/main.yml b/parcels/roles/check/check-services/tasks/main.yml new file mode 100644 index 0000000..95fa5ce --- /dev/null +++ b/parcels/roles/check/check-services/tasks/main.yml @@ -0,0 +1,67 @@ +- name: Checking Components Heartbeats + shell: "curl -s http://{{ inventory_hostname }}:8183/monitor/health | grep DOWN | grep -v grep | wc -l" + register: health + +- name: Checking Components Heartbeats + fail: + msg: "组件心跳检测异常，请通过接口 curl -s http://{{ inventory_hostname }}:8183/monitor/health 检查具体DOWN组件。" + when: health.stdout != '0' + +- name: Checking Metadata + shell: "curl -s http://{{ inventory_hostname }}:8183/diagnosis/metadata | grep '\"status\":200' | grep -v grep | wc -l" + register: metadata + +- name: Checking Metadata + fail: + msg: "元数据验证异常，请通过接口 curl -s http://{{ inventory_hostname }}:8183/diagnosis/metadata 检查具体异常信息。" + when: metadata.stdout != '1' + +- name: Checking SQL + shell: "curl -s http://{{ inventory_hostname }}:8183/diagnosis/runSql | grep '\"status\":200' | grep -v grep | wc -l" + register: sql + +- name: Checking SQL + fail: + msg: "系统内置的SQL数据集执行异常，请通过接口 curl -s http://{{ inventory_hostname }}:8183/diagnosis/runSql 检查具体异常信息。" + when: sql.stdout != '1' + +#TODO 因服务版本问题，于21.07版本开启验证 +#- name: Checking {{ groups.hos[0] }} galaxy-hos-service +# shell: "curl --location -s -w %{http_code} http://{{ groups.hos[0] }}:8186/monitor | grep 200 | wc -l" +# register: monitor_out +# run_once: true +# delegate_facts: true +# delegate_to: '{{ groups.hos[0] }}' +# +#- fail: +# msg: "HOS Monitor检测异常,请通过接口 curl --location -s -w %{http_code} http://{{ groups.hos[0] }}:8186/monitor 检查具体异常信息" +# when: monitor_out.stdout != '1' +# +#- name: Checking {{ groups.hos[1] }} galaxy-hos-service +# shell: "curl --location -s -w %{http_code} http://{{ groups.hos[1] }}:8186/monitor | grep 200 | wc -l" +# register: monitor_out +# run_once: true +# delegate_facts: true +# delegate_to: '{{ groups.hos[1] }}' +# +#- fail: +# msg: "HOS Monitor检测异常,请通过接口 curl --location -s -w %{http_code} http://{{ groups.hos[1] }}:8186/monitor 检查具体异常信息" +# when: monitor_out.stdout != '1' + +- name: wait gohangout start ,sleep 30s + shell: sleep 30 + +- name: Checking gohangout start job Num + shell: "ps -ef | grep gohangout | grep worker | grep -v grep | wc -l" + register: check_out + run_once: true + delegate_facts: true + delegate_to: '{{ groups.gohangout[1] }}' + +- fail: + msg: "{{ inventory_hostname }}- gohangout start job num is not all. Please check it" + when: check_out.stdout < '10' + run_once: true + delegate_facts: true + delegate_to: '{{ groups.gohangout[1] }}' + diff --git a/parcels/roles/components/arangodb/files/arangodb.tar.gz b/parcels/roles/components/arangodb/files/arangodb.tar.gz Binary files differnew file mode 100644 index 0000000..b65fe06 --- /dev/null +++ b/parcels/roles/components/arangodb/files/arangodb.tar.gz diff --git a/parcels/roles/components/arangodb/files/conf.bak.tar b/parcels/roles/components/arangodb/files/conf.bak.tar Binary files differnew file mode 100644 index 0000000..a7c7bb8 --- /dev/null +++ b/parcels/roles/components/arangodb/files/conf.bak.tar diff --git a/parcels/roles/components/arangodb/tasks/load_arangodb.yml b/parcels/roles/components/arangodb/tasks/load_arangodb.yml new file mode 100644 index 0000000..2d323e3 --- /dev/null +++ b/parcels/roles/components/arangodb/tasks/load_arangodb.yml @@ -0,0 +1,57 @@ +- name: stop and remove {{ arangodb_image_container_name }} container + docker_container: + name: '{{ arangodb_image_container_name }}' + state: absent + +- name: remove old {{ arangodb_image_name }} image + docker_image: + name: '{{ arangodb_image_name }}' + tag: '{{ arangodb_image_tag_name }}' + state: absent + +- name: create {{ data_path }}/{{ arangodb_soft_home_path }} + file: + state: directory + path: '{{ data_path }}/{{ arangodb_soft_home_path }}' + +- name: create {{ data_path }}/{{ arango_volume_path }} + file: + state: directory + path: '{{ data_path }}/{{ arango_volume_path }}' + + +- name: copy {{ arangodb_image_tar_name }} + copy: + src: '{{ package_path }}/{{ arangodb_image_tar_name }}' + dest: '{{ data_path }}/{{ arangodb_soft_home_path }}/{{ arangodb_image_tar_name }}' + force: true + backup: yes + +- name: copy arangodb tar file + copy: + src: '../files/arangodb.tar.gz' + dest: '{{ data_path }}/{{ arango_volume_path }}/' + force: true + backup: yes + +- name: tar -zxvf arangodb.tar.gz + shell: cd {{ data_path }}/{{ arango_volume_path }}/ && tar -zxvf arangodb.tar.gz && rm -rf arangodb.tar.gz + +- name: load new {{ arangodb_image_name }} image from {{ arangodb_image_tar_name }} + docker_image: + name: '{{ arangodb_image_name }}' + tag: '{{ arangodb_image_tag_name }}' + load_path: '{{ data_path }}/{{ arangodb_soft_home_path }}/{{ arangodb_image_tar_name }}' + source: load + force_tag: yes + force_source: yes + +- name: copy {{ arangodb_image_name }} docker-compose.yml + template: + src: docker-compose.yml.j2 + dest: '{{ data_path }}/{{ arangodb_soft_home_path }}/docker-compose.yml' + backup: yes + +- name: start {{ arangodb_image_container_name }} container + docker_compose: + project_src: '{{ data_path }}/{{ arangodb_soft_home_path }}' diff --git a/parcels/roles/components/arangodb/tasks/main.yml b/parcels/roles/components/arangodb/tasks/main.yml new file mode 100644 index 0000000..efc9898 --- /dev/null +++ b/parcels/roles/components/arangodb/tasks/main.yml @@ -0,0 +1,2 @@ +#- include: update_python3_and_docker_py.yml +- include: load_arangodb.yml diff --git a/parcels/roles/components/arangodb/templates/docker-compose.yml.j2 b/parcels/roles/components/arangodb/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..37dd384 --- /dev/null +++ b/parcels/roles/components/arangodb/templates/docker-compose.yml.j2 @@ -0,0 +1,19 @@ +version: '2' + +services: + arangodb: + image: {{ arangodb_image_name }}:{{ arangodb_image_tag_name }} + restart: always + container_name: {{ arangodb_image_container_name }} + environment: + ARANGO_ROOT_PASSWORD: "{{ arangodb_password }}" + ports: + - "8529:8529" + volumes: + - "{{ data_path }}/{{ arango_volume_path }}/conf:/etc/arangodb3" + - "{{ data_path }}/{{ arango_volume_path }}/data:/var/lib/arangodb3" + - "{{ data_path }}/{{ arango_volume_path }}/log:/var/log/arangodb3" + - "/etc/localtime:/etc/localtime:ro" + - "/etc/timezone:/etc/timezone:ro" + restart: always + network_mode: "host" diff --git a/parcels/roles/components/arangodb/vars/main.yml b/parcels/roles/components/arangodb/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/components/arangodb/vars/main.yml diff --git a/parcels/roles/components/clickhouse/common/files/ckdata_list b/parcels/roles/components/clickhouse/common/files/ckdata_list new file mode 100644 index 0000000..fd51a19 --- /dev/null +++ b/parcels/roles/components/clickhouse/common/files/ckdata_list @@ -0,0 +1 @@ +192.168.44.35 diff --git a/parcels/roles/components/clickhouse/common/files/ckquery_list b/parcels/roles/components/clickhouse/common/files/ckquery_list new file mode 100644 index 0000000..a80722f --- /dev/null +++ b/parcels/roles/components/clickhouse/common/files/ckquery_list @@ -0,0 +1,2 @@ +192.168.44.33 +192.168.44.34 diff --git a/parcels/roles/components/clickhouse/common/tasks/install_clickhouse.yml b/parcels/roles/components/clickhouse/common/tasks/install_clickhouse.yml new file mode 100644 index 0000000..0fb6599 --- /dev/null +++ b/parcels/roles/components/clickhouse/common/tasks/install_clickhouse.yml @@ -0,0 +1,172 @@ +- name: Check whether the CPU supports the SSE4_2 instruction set + shell: lscpu |grep sse4_2 | wc -l + register: check_cpu_out + +- fail: + msg: "检测目标服务器集群内存在CPU不支持SSE4_2指令集的节点，请检查" + when: check_cpu_out.stdout < '1' + +- block: + - name: Check if the Clickhouse service already exists + shell: rpm -qa | grep clickhouse | wc -l + register: check_out + - name: copy unload_ck.sh to {{ install_path }}/ + template: + src: 'unload_ck.sh.j2' + dest: '{{ install_path }}/unload_ck.sh' + force: true + mode: 0755 + when: check_out.stdout >= '1' + - name: unload kafka + shell: cd {{ install_path }} && sh unload_ck.sh + when: check_out.stdout >= '1' + - name: Ansible delete {{ install_path }}/unload_ck.sh + file: + path: "{{ install_path }}/unload_ck.sh" + state: absent + - name: Check if the Clickhouse service already exists + shell: rpm -qa | grep clickhouse | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "卸载失败，组件可能非本安装部署，请手动卸载后继续安装" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "yes" + +- block: + - name: Check if the Clickhouse service already exists + shell: rpm -qa | grep clickhouse | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "检测到目标集群内已有Clickhouse请检查IP是否正确或停止/卸载已启动的组件" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "no" + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181" + when: '(groups.zookeeper|length) == 3' + run_once: true + delegate_to: 127.0.0.1 + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181,{{groups.zookeeper[3]}}:2181,{{groups.zookeeper[4]}}:2181" + when: '(groups.zookeeper|length) >= 5' + run_once: true + delegate_to: 127.0.0.1 + +- name: create clickhouse package path:{{ install_path }}/clickhouse + file: + state: directory + path: '{{ install_path }}/clickhouse' + +- name: create clickhouse rpm path:{{ install_path }}/clickhouse/rpm + file: + state: directory + path: '{{ install_path }}/clickhouse/rpm' + +- name: create clickhouse rpm path:{{ install_path }}/clickhouse/logs + file: + state: directory + path: '{{ install_path }}/clickhouse/logs' + +- name: create clickhouse rpm path:{{ install_path }}/clickhouse-exporter + file: + state: directory + path: '{{ install_path }}/clickhouse-exporter' + +- name: create clickhouse server path:{{ install_path }}/clickhouse/clickhouse-server + file: + state: directory + path: '{{ install_path }}/clickhouse/clickhouse-server' + +#复制tar到目标服务器 +- name: copy rpm files to {{ install_path }}/clickhouse/rpm/ + copy: + src: '{{ package_path }}/clickhouse/{{ item.file }}' + dest: '{{ install_path }}/clickhouse/rpm/' + force: true + with_items: + - { file: 'clickhouse-client-{{ clickhouse_version }}-1.el7.x86_64.rpm' } + - { file: 'clickhouse-common-static-{{ clickhouse_version }}-1.el7.x86_64.rpm' } + - { file: 'clickhouse-server-{{ clickhouse_version }}-1.el7.x86_64.rpm' } + - { file: 'clickhouse-server-common-{{ clickhouse_version }}-1.el7.x86_64.rpm' } + - { file: 'unixODBC-2.3.1-11.el7.x86_64.rpm' } + +- name: get readonly key sha256 + shell: echo -n "{{ readonly_key }}"|sha256sum | tr -d '-' | sed -e 's/^[ ]*//g' | sed -e 's/[ ]*$//g' + register: readonly_key_out + +- name: readonly_key_sha to ansible variable + set_fact: readonly_key_sha="{{readonly_key_out.stdout}}" + +- name: get root key sha256 + shell: echo -n "{{ root_key }}"|sha256sum| tr -d '-' | sed -e 's/^[ ]*//g' | sed -e 's/[ ]*$//g' + register: root_key_out + +- name: root_key_sha to ansible variable + set_fact: root_key_sha="{{root_key_out.stdout}}" + +#拷贝clickhouse 配置文件 +- name: copy clickhouse config files to {{ install_path }}/clickhouse/clickhouse-server/ + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + mode: 0755 + force: true + with_items: + - { src: 'clickhouse-server.j2', dest: '{{ install_path }}/clickhouse/clickhouse-server/clickhouse-server' } + - { src: 'set_zk_node.sh.j2', dest: '{{ install_path }}/clickhouse/clickhouse-server/set_zk_node.sh' } + - { src: 'clickhouse-server.j2', dest: '/etc/init.d/clickhouse-server' } + - { src: 'ckquery_list.j2', dest: '{{ install_path }}/clickhouse/clickhouse-server/ckquery_list' } + - { src: 'ckdata_list.j2', dest: '{{ install_path }}/clickhouse/clickhouse-server/ckdata_list' } + +#拷贝clickhouse 配置文件 +- name: copy clickhouse config files to {{ install_path }}/clickhouse/clickhouse-server/ + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + force: true + backup: yes + with_items: + - { src: 'config.xml.j2', dest: '{{ install_path }}/clickhouse/clickhouse-server/config.xml' } + - { src: 'users.xml.j2', dest: '{{ install_path }}/clickhouse/clickhouse-server/users.xml' } + +- name: check unixODBC + shell: rpm -qa | grep "unixODBC-2.3.1-11.el7.x86_64" | wc -l + register: odbc_install + +- name: install unixODBC + shell: rpm -ivh {{ install_path }}/clickhouse/rpm/unixODBC-2.3.1-11.el7.x86_64.rpm + when: odbc_install.stdout == '0' + +- name: copy clickhouse-exporter.tar to {{ install_path }}/clickhouse-exporter/ + copy: + src: '{{ package_path }}/clickhouse/clickhouse-exporter.tar' + dest: '{{ install_path }}/clickhouse-exporter/' + force: true + +- name: copy start_exporter.sh to {{ install_path }}/clickhouse-exporter/ + template: + src: 'start_exporter.sh.j2' + dest: '{{ install_path }}/clickhouse-exporter/start_exporter.sh' + force: true + mode: 0755 + +#导入exporter镜像 +- name: load clickhouse-expoter + docker_image: + name: 'clickhouse-exporter' + tag: 'v1.0' + load_path: '{{ install_path }}/clickhouse-exporter/clickhouse-exporter.tar' + source: load + force_tag: yes + force_source: yes + +#启动exporter +- name: start clickhouse_exporter + shell: cd {{ install_path }}/clickhouse-exporter/ && ./start_exporter.sh diff --git a/parcels/roles/components/clickhouse/common/tasks/main.yml b/parcels/roles/components/clickhouse/common/tasks/main.yml new file mode 100644 index 0000000..dceb53f --- /dev/null +++ b/parcels/roles/components/clickhouse/common/tasks/main.yml @@ -0,0 +1 @@ +- include: install_clickhouse.yml diff --git a/parcels/roles/components/clickhouse/common/templates/ckdata_list.j2 b/parcels/roles/components/clickhouse/common/templates/ckdata_list.j2 new file mode 100644 index 0000000..84337c8 --- /dev/null +++ b/parcels/roles/components/clickhouse/common/templates/ckdata_list.j2 @@ -0,0 +1,5 @@ +{% for ip in ansible_play_hosts %} +{% if (groups.clickhouse[0]) != ip and (groups.clickhouse[1]) != ip%} +{{ip}} +{% endif %} +{% endfor %} diff --git a/parcels/roles/components/clickhouse/common/templates/ckquery_list.j2 b/parcels/roles/components/clickhouse/common/templates/ckquery_list.j2 new file mode 100644 index 0000000..b9ffcb8 --- /dev/null +++ b/parcels/roles/components/clickhouse/common/templates/ckquery_list.j2 @@ -0,0 +1,2 @@ +{{ groups.clickhouse[0] }} +{{ groups.clickhouse[1] }} diff --git a/parcels/roles/components/clickhouse/common/templates/clickhouse-server.j2 b/parcels/roles/components/clickhouse/common/templates/clickhouse-server.j2 new file mode 100755 index 0000000..faee47f --- /dev/null +++ b/parcels/roles/components/clickhouse/common/templates/clickhouse-server.j2 @@ -0,0 +1,355 @@ +#!/bin/sh +### BEGIN INIT INFO +# Provides: clickhouse-server +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Required-Start: +# Required-Stop: +# Short-Description: Yandex clickhouse-server daemon +### END INIT INFO + + +CLICKHOUSE_USER=clickhouse +CLICKHOUSE_GROUP=${CLICKHOUSE_USER} +SHELL=/bin/bash +PROGRAM=clickhouse-server +GENERIC_PROGRAM=clickhouse +EXTRACT_FROM_CONFIG=${GENERIC_PROGRAM}-extract-from-config +SYSCONFDIR={{ install_path }}/clickhouse/$PROGRAM +CLICKHOUSE_LOGDIR={{ install_path }}/clickhouse/clickhouse-server +CLICKHOUSE_LOGDIR_USER=root +CLICKHOUSE_DATADIR_OLD={{ install_path }}/clickhouse/clickhouse_old +LOCALSTATEDIR=/var/lock +BINDIR=/usr/bin +CLICKHOUSE_CRONFILE=/etc/cron.d/clickhouse-server +CLICKHOUSE_CONFIG={{ install_path }}/clickhouse/clickhouse-server/config.xml +LOCKFILE=$LOCALSTATEDIR/$PROGRAM +RETVAL=0 +CLICKHOUSE_PIDDIR={{ install_path }}/clickhouse/$PROGRAM +CLICKHOUSE_PIDFILE="$CLICKHOUSE_PIDDIR/$PROGRAM.pid" + +# Some systems lack "flock" +command -v flock >/dev/null && FLOCK=flock + + +# Override defaults from optional config file +test -f /etc/default/clickhouse && . /etc/default/clickhouse + +# On x86_64, check for required instruction set. +if uname -mpi | grep -q 'x86_64'; then + if ! grep -q 'sse4_2' /proc/cpuinfo; then + # On KVM, cpuinfo could falsely not report SSE 4.2 support, so skip the check. + if ! grep -q 'Common KVM processor' /proc/cpuinfo; then + + # Some other VMs also report wrong flags in cpuinfo. + # Tricky way to test for instruction set: + # create temporary binary and run it; + # if it get caught illegal instruction signal, + # then required instruction set is not supported really. + # + # Generated this way: + # gcc -xc -Os -static -nostdlib - <<< 'void _start() { __asm__("pcmpgtq %%xmm0, %%xmm1; mov $0x3c, %%rax; xor %%rdi, %%rdi; syscall":::"memory"); }' && strip -R .note.gnu.build-id -R .comment -R .eh_frame -s ./a.out && gzip -c -9 ./a.out | base64 -w0; echo + + if ! (echo -n 'H4sICAwAW1cCA2Eub3V0AKt39XFjYmRkgAEmBjsGEI+H0QHMd4CKGyCUAMUsGJiBJDNQNUiYlQEZOKDQclB9cnD9CmCSBYqJBRxQOvBpSQobGfqIAWn8FuYnPI4fsAGyPQz/87MeZtArziguKSpJTGLQK0mtKGGgGHADMSgoYH6AhTMPNHyE0NQzYuEzYzEXFr6CBPQDANAsXKTwAQAA' | base64 -d | gzip -d > /tmp/clickhouse_test_sse42 && chmod a+x /tmp/clickhouse_test_sse42 && /tmp/clickhouse_test_sse42); then + echo 'Warning! SSE 4.2 instruction set is not supported' + #exit 3 + fi + fi + fi +fi + + +SUPPORTED_COMMANDS="{start|stop|status|restart|forcestop|forcerestart|reload|condstart|condstop|condrestart|condreload|initdb}" +is_supported_command() +{ + echo "$SUPPORTED_COMMANDS" | grep -E "(\{|\|)$1(\||})" &> /dev/null +} + + +is_running() +{ + [ -r "$CLICKHOUSE_PIDFILE" ] && pgrep -s $(cat "$CLICKHOUSE_PIDFILE") 1> /dev/null 2> /dev/null +} + + +wait_for_done() +{ + while is_running; do + sleep 1 + done +} + + +die() +{ + echo $1 >&2 + exit 1 +} + + +# Check that configuration file is Ok. +check_config() +{ + if [ -x "$BINDIR/$EXTRACT_FROM_CONFIG" ]; then + su -s $SHELL ${CLICKHOUSE_USER} -c "$BINDIR/$EXTRACT_FROM_CONFIG --config-file=\"$CLICKHOUSE_CONFIG\" --key=path" >/dev/null || die "Configuration file ${CLICKHOUSE_CONFIG} doesn't parse successfully. Won't restart server. You may use forcerestart if you are sure."; + fi +} + + +initdb() +{ + if [ -d ${SYSCONFDIR} ]; then + su -s /bin/sh ${CLICKHOUSE_USER} -c "test -w ${SYSCONFDIR}" || chown ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${SYSCONFDIR} + fi + + if [ -x "$BINDIR/$EXTRACT_FROM_CONFIG" ]; then + CLICKHOUSE_DATADIR_FROM_CONFIG=$(su -s $SHELL ${CLICKHOUSE_USER} -c "$BINDIR/$EXTRACT_FROM_CONFIG --config-file=\"$CLICKHOUSE_CONFIG\" --key=path") + if [ "(" "$?" -ne "0" ")" -o "(" -z "${CLICKHOUSE_DATADIR_FROM_CONFIG}" ")" ]; then + die "Cannot obtain value of path from config file: ${CLICKHOUSE_CONFIG}"; + fi + echo "Path to data directory in ${CLICKHOUSE_CONFIG}: ${CLICKHOUSE_DATADIR_FROM_CONFIG}" + else + CLICKHOUSE_DATADIR_FROM_CONFIG="/var/lib/clickhouse" + fi + + if ! getent group ${CLICKHOUSE_USER} >/dev/null; then + echo "Can't chown to non-existing user ${CLICKHOUSE_USER}" + return + fi + if ! getent passwd ${CLICKHOUSE_GROUP} >/dev/null; then + echo "Can't chown to non-existing group ${CLICKHOUSE_GROUP}" + return + fi + + if ! $(su -s $SHELL ${CLICKHOUSE_USER} -c "test -r ${CLICKHOUSE_CONFIG}"); then + echo "Warning! clickhouse config [${CLICKHOUSE_CONFIG}] not readable by user [${CLICKHOUSE_USER}]" + fi + + if ! $(su -s $SHELL ${CLICKHOUSE_USER} -c "test -O \"${CLICKHOUSE_DATADIR_FROM_CONFIG}\" && test -G \"${CLICKHOUSE_DATADIR_FROM_CONFIG}\""); then + if [ $(dirname "${CLICKHOUSE_DATADIR_FROM_CONFIG}") == "/" ]; then + echo "Directory ${CLICKHOUSE_DATADIR_FROM_CONFIG} seems too dangerous to chown." + else + if [ ! -e "${CLICKHOUSE_DATADIR_FROM_CONFIG}" ]; then + echo "Creating directory ${CLICKHOUSE_DATADIR_FROM_CONFIG}" + mkdir -p "${CLICKHOUSE_DATADIR_FROM_CONFIG}" + fi + + echo "Changing owner of [${CLICKHOUSE_DATADIR_FROM_CONFIG}] to [${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP}]" + chown -R ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} "${CLICKHOUSE_DATADIR_FROM_CONFIG}" + fi + fi + + if ! $(su -s $SHELL ${CLICKHOUSE_USER} -c "test -w ${CLICKHOUSE_LOGDIR}"); then + echo "Changing owner of [${CLICKHOUSE_LOGDIR}/*] to [${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP}]" + chown -R ${CLICKHOUSE_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR}/* + echo "Changing owner of [${CLICKHOUSE_LOGDIR}] to [${CLICKHOUSE_LOGDIR_USER}:${CLICKHOUSE_GROUP}]" + chown ${CLICKHOUSE_LOGDIR_USER}:${CLICKHOUSE_GROUP} ${CLICKHOUSE_LOGDIR} + fi +} + + +start() +{ + [ -x $BINDIR/$PROGRAM ] || exit 0 + local EXIT_STATUS + EXIT_STATUS=0 + + echo -n "Start $PROGRAM service: " + + if is_running; then + echo -n "already running " + EXIT_STATUS=1 + else + ulimit -n 262144 + mkdir -p $CLICKHOUSE_PIDDIR + chown -R $CLICKHOUSE_USER:$CLICKHOUSE_GROUP $CLICKHOUSE_PIDDIR + initdb + if ! is_running; then + # Lock should not be held while running child process, so we release the lock. Note: obviously, there is race condition. + # But clickhouse-server has protection from simultaneous runs with same data directory. + su -s $SHELL ${CLICKHOUSE_USER} -c "$FLOCK -u 9; exec -a \"$PROGRAM\" \"$BINDIR/$PROGRAM\" --daemon --pid-file=\"$CLICKHOUSE_PIDFILE\" --config-file=\"$CLICKHOUSE_CONFIG\"" + EXIT_STATUS=$? + if [ $EXIT_STATUS -ne 0 ]; then + break + fi + fi + fi + + if [ $EXIT_STATUS -eq 0 ]; then + echo "DONE" + else + echo "FAILED" + fi + + return $EXIT_STATUS +} + + +stop() +{ + local EXIT_STATUS + EXIT_STATUS=0 + + if [ -f $CLICKHOUSE_PIDFILE ]; then + + echo -n "Stop $PROGRAM service: " + + kill -TERM $(cat "$CLICKHOUSE_PIDFILE") + + wait_for_done + + echo "DONE" + fi + return $EXIT_STATUS +} + + +restart() +{ + check_config + stop + start +} + + +forcestop() +{ + local EXIT_STATUS + EXIT_STATUS=0 + + echo -n "Stop forcefully $PROGRAM service: " + + kill -KILL $(cat "$CLICKHOUSE_PIDFILE") + + wait_for_done + + echo "DONE" + return $EXIT_STATUS +} + + +forcerestart() +{ + forcestop + start +} + +use_cron() +{ + # 1. running systemd + if [ -x "/bin/systemctl" ] && [ -f /etc/systemd/system/clickhouse-server.service ] && [ -d /run/systemd/system ]; then + return 1 + fi + # 2. disabled by config + if [ -z "$CLICKHOUSE_CRONFILE" ]; then + return 2 + fi + return 0 +} + +enable_cron() +{ + use_cron && sed -i 's/^#*//' "$CLICKHOUSE_CRONFILE" +} + + +disable_cron() +{ + use_cron && sed -i 's/^#*/#/' "$CLICKHOUSE_CRONFILE" +} + + +is_cron_disabled() +{ + use_cron || return 0 + + # Assumes that either no lines are commented or all lines are commented. + # Also please note, that currently cron file for ClickHouse has only one line (but some time ago there was more). + grep -q -E '^#' "$CLICKHOUSE_CRONFILE"; +} + + +main() +{ + # See how we were called. + EXIT_STATUS=0 + case "$1" in + start) + start && enable_cron + ;; + stop) + disable_cron && stop + ;; + restart) + restart && enable_cron + ;; + forcestop) + disable_cron && forcestop + ;; + forcerestart) + forcerestart && enable_cron + ;; + reload) + restart + ;; + condstart) + is_running || start + ;; + condstop) + is_running && stop + ;; + condrestart) + is_running && restart + ;; + condreload) + is_running && restart + ;; + initdb) + initdb + ;; + enable_cron) + enable_cron + ;; + disable_cron) + disable_cron + ;; + *) + echo "Usage: $0 $SUPPORTED_COMMANDS" + exit 2 + ;; + esac + + exit $EXIT_STATUS +} + + +status() +{ + if is_running; then + echo "$PROGRAM service is running" + else + if is_cron_disabled; then + echo "$PROGRAM service is stopped"; + else + echo "$PROGRAM: process unexpectedly terminated" + fi + fi +} + + +# Running commands without need of locking +case "$1" in +status) + status + exit 0 + ;; +esac + + +( + if $FLOCK -n 9; then + main "$@" + else + echo "Init script is already running" && exit 1 + fi +) 9> $LOCKFILE diff --git a/parcels/roles/components/clickhouse/common/templates/config.xml.j2 b/parcels/roles/components/clickhouse/common/templates/config.xml.j2 new file mode 100644 index 0000000..fe3ddc4 --- /dev/null +++ b/parcels/roles/components/clickhouse/common/templates/config.xml.j2 @@ -0,0 +1,373 @@ +<?xml version="1.0"?> +<yandex> + <logger> + <!-- Possible levels: https://github.com/pocoproject/poco/blob/develop/Foundation/include/Poco/Logger.h#L105 --> + <level>error</level> + <log>{{ install_path }}/clickhouse/logs/clickhouse-server.log</log> + <errorlog>{{ install_path }}/clickhouse/logs/clickhouse-server.err.log</errorlog> + <size>200M</size> + <count>10</count> + <!-- <console>1</console> --> <!-- Default behavior is autodetection (log to console if not daemon mode and is tty) --> + </logger> + <!--display_name>production</display_name--> <!-- It is the name that will be shown in the client --> + <http_port>8123</http_port> + <tcp_port>9001</tcp_port> + + <!-- For HTTPS and SSL over native protocol. --> + <!-- + <https_port>8443</https_port> + <tcp_port_secure>9440</tcp_port_secure> + --> + + <!-- Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 --> + <openSSL> + <server> <!-- Used for https server AND secure tcp port --> + <!-- openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt --> + <certificateFile>{{ install_path }}/clickhouse/clickhouse-server/server.crt</certificateFile> + <privateKeyFile>{{ install_path }}/clickhouse/clickhouse-server/server.key</privateKeyFile> + <!-- openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096 --> + <dhParamsFile>{{ install_path }}/clickhouse/clickhouse-server/dhparam.pem</dhParamsFile> + <verificationMode>none</verificationMode> + <loadDefaultCAFile>true</loadDefaultCAFile> + <cacheSessions>true</cacheSessions> + <disableProtocols>sslv2,sslv3</disableProtocols> + <preferServerCiphers>true</preferServerCiphers> + </server> + + <client> <!-- Used for connecting to https dictionary source --> + <loadDefaultCAFile>true</loadDefaultCAFile> + <cacheSessions>true</cacheSessions> + <disableProtocols>sslv2,sslv3</disableProtocols> + <preferServerCiphers>true</preferServerCiphers> + <!-- Use for self-signed: <verificationMode>none</verificationMode> --> + <invalidCertificateHandler> + <!-- Use for self-signed: <name>AcceptCertificateHandler</name> --> + <name>RejectCertificateHandler</name> + </invalidCertificateHandler> + </client> + </openSSL> + + <!-- Default root page on http[s] server. For example load UI from https://tabix.io/ when opening http://localhost:8123 --> + <!-- + <http_server_default_response><![CDATA[<html ng-app="SMI2"><head><base href="http://ui.tabix.io/"></head><body><div ui-view="" class="content-ui"></div><script src="http://loader.tabix.io/master.js"></script></body></html>]]></http_server_default_response> + --> + + <!-- Port for communication between replicas. Used for data exchange. --> + <interserver_http_port>9009</interserver_http_port> + + <!-- Hostname that is used by other replicas to request this server. + If not specified, than it is determined analoguous to 'hostname -f' command. + This setting could be used to switch replication to another network interface. + --> + + <interserver_http_host>{{ inventory_hostname }}</interserver_http_host> + + + <!-- Listen specified host. use :: (wildcard IPv6 address), if you want to accept connections both with IPv4 and IPv6 from everywhere. --> + <listen_host>0.0.0.0</listen_host> + <!-- Same for hosts with disabled ipv6: --> + <!--<listen_host>0.0.0.0</listen_host>--> + + <!-- Default values - try listen localhost on ipv4 and ipv6: --> + +<!--<listen_host>::1</listen_host>--> + <!-- <listen_host>127.0.0.1</listen_host>--> + + <!-- Don't exit if ipv6 or ipv4 unavailable, but listen_host with this protocol specified --> + <!-- <listen_try>0</listen_try>--> + + <!-- Allow listen on same address:port --> + <!-- <listen_reuse_port>0</listen_reuse_port>--> + + <listen_backlog>64</listen_backlog> + + <max_connections>4096</max_connections> + <keep_alive_timeout>600</keep_alive_timeout> + + <!-- Maximum number of concurrent queries. --> + <max_concurrent_queries>150</max_concurrent_queries> + + <!-- Set limit on number of open files (default: maximum). This setting makes sense on Mac OS X because getrlimit() fails to retrieve + correct maximum value. --> + <!-- <max_open_files>262144</max_open_files> --> + + <!-- Size of cache of uncompressed blocks of data, used in tables of MergeTree family. + In bytes. Cache is single for server. Memory is allocated only on demand. + Cache is used when 'use_uncompressed_cache' user setting turned on (off by default). + Uncompressed cache is advantageous only for very short queries and in rare cases. + --> + <uncompressed_cache_size>8589934592</uncompressed_cache_size> + + <!-- Approximate size of mark cache, used in tables of MergeTree family. + In bytes. Cache is single for server. Memory is allocated only on demand. + You should not lower this value. + --> + <mark_cache_size>5368709120</mark_cache_size> + + + <!-- Path to data directory, with trailing slash. --> + <path>{{ data_path }}/clickhouse/</path> + + <!-- Path to temporary data for processing hard queries. --> + <tmp_path>{{ data_path }}/clickhouse/tmp/</tmp_path> + + <!-- Directory with user provided files that are accessible by 'file' table function. --> + <user_files_path>{{ install_path }}/clickhouse/user_files/</user_files_path> + + <!-- Path to configuration file with users, access rights, profiles of settings, quotas. --> + <users_config>users.xml</users_config> + + <!-- Default profile of settings. --> + <default_profile>default</default_profile> + + <!-- System profile of settings. This settings are used by internal processes (Buffer storage, Distibuted DDL worker and so on). --> + <!-- <system_profile>default</system_profile> --> + + <!-- Default database. --> + <default_database>{{ default_database }}</default_database> + + <!-- Server time zone could be set here. + + Time zone is used when converting between String and DateTime types, + when printing DateTime in text formats and parsing DateTime from text, + it is used in date and time related functions, if specific time zone was not passed as an argument. + + Time zone is specified as identifier from IANA time zone database, like UTC or Africa/Abidjan. + If not specified, system time zone at server startup is used. + + Please note, that server could display time zone alias instead of specified name. + Example: W-SU is an alias for Europe/Moscow and Zulu is an alias for UTC. + --> + <!-- <timezone>Europe/Moscow</timezone> --> + + <!-- You can specify umask here (see "man umask"). Server will apply it on startup. + Number is always parsed as octal. Default umask is 027 (other users cannot read logs, data files, etc; group can only read). + --> + <!-- <umask>022</umask> --> + + <!-- Configuration of clusters that could be used in Distributed tables. + https://clickhouse.yandex/docs/en/table_engines/distributed/ + --> + <remote_servers incl="clickhouse_remote_servers" > + <!-- Test only shard config for testing distributed storage + <test_shard_localhost> + <shard> + <replica> + <host>localhost</host> + <port>9000</port> + </replica> + </shard> + </test_shard_localhost> + <test_shard_localhost_secure> + <shard> + <replica> + <host>localhost</host> + <port>9440</port> + <secure>1</secure> + </replica> + </shard> + </test_shard_localhost_secure>--> + </remote_servers> + + + <!-- If element has 'incl' attribute, then for it's value will be used corresponding substitution from another file. + By default, path to file with substitutions is /etc/metrika.xml. It could be changed in config in 'include_from' element. + Values for substitutions are specified in /yandex:wq +/name_of_substitution elements in that file. + --> + + <!-- ZooKeeper is used to store metadata about replicas, when using Replicated tables. + Optional. If you don't use replicated tables, you could omit that. + + See https://clickhouse.yandex/docs/en/table_engines/replication/ + --> + <zookeeper incl="zookeeper-servers" optional="true" /> + + <!-- Substitutions for parameters of replicated tables. + Optional. If you don't use replicated tables, you could omit that. + + See https://clickhouse.yandex/docs/en/table_engines/replication/#creating-replicated-tables + --> + <macros incl="macros" optional="true" /> + + + <!-- Reloading interval for embedded dictionaries, in seconds. Default: 3600. --> + <builtin_dictionaries_reload_interval>3600</builtin_dictionaries_reload_interval> + + + <!-- Maximum session timeout, in seconds. Default: 3600. --> + <max_session_timeout>21600</max_session_timeout> + + <!-- Default session timeout, in seconds. Default: 60. --> + <default_session_timeout>6000</default_session_timeout> +<max_table_size_to_drop>0</max_table_size_to_drop> +<max_partition_size_to_drop>0</max_partition_size_to_drop> +<include_from>{{ install_path }}/clickhouse/clickhouse-server/metrika.xml</include_from> + <!-- Sending data to Graphite for monitoring. Several sections can be defined. --> + <!-- + interval - send every X second + root_path - prefix for keys + hostname_in_path - append hostname to root_path (default = true) + metrics - send data from table system.metrics + events - send data from table system.events + asynchronous_metrics - send data from table system.asynchronous_metrics + --> + <!-- + <graphite> + <host>localhost</host> + <port>42000</port> + <timeout>0.1</timeout> + <interval>60</interval> + <root_path>one_min</root_path> + <hostname_in_path>true</hostname_in_path> + + <metrics>true</metrics> + <events>true</events> + <asynchronous_metrics>true</asynchronous_metrics> + </graphite> + <graphite> + <host>localhost</host> + <port>42000</port> + <timeout>0.1</timeout> + <interval>1</interval> + <root_path>one_sec</root_path> + + <metrics>true</metrics> + <events>true</events> + <asynchronous_metrics>false</asynchronous_metrics> + </graphite> + --> + + + <!-- Query log. Used only for queries with setting log_queries = 1. --> + <query_log> + <!-- What table to insert data. If table is not exist, it will be created. + When query log structure is changed after system update, + then old table will be renamed and new table will be created automatically. + --> + <database>system</database> + <table>query_log</table> + <!-- + PARTITION BY expr https://clickhouse.yandex/docs/en/table_engines/custom_partitioning_key/ + Example: + event_date + toMonday(event_date) + toYYYYMM(event_date) + toStartOfHour(event_time) + --> + <partition_by>toYYYYMM(event_date)</partition_by> + <!-- Interval of flushing data. --> + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </query_log> + + + <!-- Uncomment if use part_log + <part_log> + <database>system</database> + <table>part_log</table> + + <flush_interval_milliseconds>7500</flush_interval_milliseconds> + </part_log> + --> + + + <!-- Parameters for embedded dictionaries, used in Yandex.Metrica. + See https://clickhouse.yandex/docs/en/dicts/internal_dicts/ + --> + + <!-- Path to file with region hierarchy. --> + <!-- <path_to_regions_hierarchy_file>/opt/geo/regions_hierarchy.txt</path_to_regions_hierarchy_file> --> + + <!-- Path to directory with files containing names of regions --> + <!-- <path_to_regions_names_files>/opt/geo/</path_to_regions_names_files> --> + + + <!-- Configuration of external dictionaries. See: + https://clickhouse.yandex/docs/en/dicts/external_dicts/ + --> + <dictionaries_config>*_dictionary.xml</dictionaries_config> + + <!-- Uncomment if you want data to be compressed 30-100% better. + Don't do that if you just started using ClickHouse. + --> + <compression incl="clickhouse_compression"> + <!-- + <!- - Set of variants. Checked in order. Last matching case wins. If nothing matches, lz4 will be used. - -> + <case> + + <!- - Conditions. All must be satisfied. Some conditions may be omitted. - -> + <min_part_size>10000000000</min_part_size> <!- - Min part size in bytes. - -> + <min_part_size_ratio>0.01</min_part_size_ratio> <!- - Min size of part relative to whole table size. - -> + + <!- - What compression method to use. - -> + <method>zstd</method> + </case> + --> + </compression> + + <!-- Allow to execute distributed DDL queries (CREATE, DROP, ALTER, RENAME) on cluster. + Works only if ZooKeeper is enabled. Comment it if such functionality isn't required. --> + <distributed_ddl> + <!-- Path in ZooKeeper to queue with DDL queries --> + <path>/clickhouse/task_queue/ddl</path> + + <!-- Settings from this profile will be used to execute DDL queries --> + <!-- <profile>default</profile> --> + </distributed_ddl> + + <!-- Settings to fine tune MergeTree tables. See documentation in source code, in MergeTreeSettings.h --> + <!-- + <merge_tree> + <max_suspicious_broken_parts>5</max_suspicious_broken_parts> + </merge_tree> + --> + + <!-- Protection from accidental DROP. + If size of a MergeTree table is greater than max_table_size_to_drop (in bytes) than table could not be dropped with any DROP query. + If you want do delete one table and don't want to restart clickhouse-server, you could create special file <clickhouse-path>/flags/force_drop_table and make DROP once. + By default max_table_size_to_drop is 50GB; max_table_size_to_drop=0 allows to DROP any tables. + The same for max_partition_size_to_drop. + Uncomment to disable protection. + --> + <!-- <max_table_size_to_drop>0</max_table_size_to_drop> --> + <!-- <max_partition_size_to_drop>0</max_partition_size_to_drop> --> + + <!-- Example of parameters for GraphiteMergeTree table engine --> + <graphite_rollup_example> + <pattern> + <regexp>click_cost</regexp> + <function>any</function> + <retention> + <age>0</age> + <precision>3600</precision> + </retention> + <retention> + <age>86400</age> + <precision>60</precision> + </retention> + </pattern> + <default> + <function>max</function> + <retention> + <age>0</age> + <precision>60</precision> + </retention> + <retention> + <age>3600</age> + <precision>300</precision> + </retention> + <retention> + <age>86400</age> + <precision>3600</precision> + </retention> + </default> + </graphite_rollup_example> + + <!-- Directory in <clickhouse-path> containing schema files for various input formats. + The directory will be created if it doesn't exist. + --> + <format_schema_path>{{ install_path }}/clickhouse/format_schemas/</format_schema_path> + + <!-- Uncomment to disable ClickHouse internal DNS caching. --> + <!-- <disable_internal_dns_cache>1</disable_internal_dns_cache> --> +</yandex> diff --git a/parcels/roles/components/clickhouse/common/templates/set_zk_node.sh.j2 b/parcels/roles/components/clickhouse/common/templates/set_zk_node.sh.j2 new file mode 100755 index 0000000..1002c67 --- /dev/null +++ b/parcels/roles/components/clickhouse/common/templates/set_zk_node.sh.j2 @@ -0,0 +1,23 @@ +#!/bin/bash + + +#<node index="1"> +#<host>192.168.44.33</host> +#<port>2181</port> +#</node> + +zookeeper_servers={{ zookeeper_servers }} + +start_num=`cat -n {{ install_path }}/clickhouse/clickhouse-server/metrika.xml |grep "zookeeper-servers" |awk '{print $1}'|sed -n "1"p` + +zk_node=(${zookeeper_servers//,/ }) + +index=1 +for node in ${zk_node[@]} +do + ipport=(${node//:/ }) + sed -i ''$start_num'a<node index="'$index'">\n<host>'${ipport[0]}'</host>\n<port>'${ipport[1]}'</port>\n</node>\n' {{ install_path }}/clickhouse/clickhouse-server/metrika.xml + index=`expr $index + 1` + start_num=`expr $start_num + 5` +done + diff --git a/parcels/roles/components/clickhouse/common/templates/start_exporter.sh.j2 b/parcels/roles/components/clickhouse/common/templates/start_exporter.sh.j2 new file mode 100755 index 0000000..a16e3b5 --- /dev/null +++ b/parcels/roles/components/clickhouse/common/templates/start_exporter.sh.j2 @@ -0,0 +1,9 @@ +#!/bin/bash +#每台都要启动，修改后面clickhouse节点地址与启动服务器IP对应 + + +docker rm -f clickhouse_exporter + +sleep 3 + +docker run -d -p 9904:9116 --name clickhouse_exporter --restart always --env CLICKHOUSE_USER=default --env CLICKHOUSE_PASSWORD=ceiec2019 clickhouse-exporter:v1.0 -scrape_uri=http://{{ inventory_hostname }}:8123/ diff --git a/parcels/roles/components/clickhouse/common/templates/unload_ck.sh.j2 b/parcels/roles/components/clickhouse/common/templates/unload_ck.sh.j2 new file mode 100644 index 0000000..460138e --- /dev/null +++ b/parcels/roles/components/clickhouse/common/templates/unload_ck.sh.j2 @@ -0,0 +1,28 @@ +#!/bin/bash +source /etc/profile + +function killService(){ +keeppath='/etc/init.d/clickhouse-server' +if [ -x $keeppath ];then +service clickhouse-server stop +fi +} + +function killPid(){ + +livenum=`rpm -qa | grep clickhouse | wc -l` +if [ $livenum -ne 0 ];then +service clickhouse-server stop +rpm -e --noscripts clickhouse-client-20.3.12.112-1.el7.x86_64 +rpm -e --noscripts clickhouse-server-20.3.12.112-1.el7.x86_64 +rpm -e --noscripts clickhouse-server-common-20.3.12.112-1.el7.x86_64 +rpm -e --noscripts clickhouse-common-static-20.3.12.112-1.el7.x86_64 +fi + +} + +killService +sleep 15 +killPid +rm -rf {{ install_path }}/clickhouse +rm -rf {{ data_path }}/clickhouse diff --git a/parcels/roles/components/clickhouse/common/templates/users.xml.j2 b/parcels/roles/components/clickhouse/common/templates/users.xml.j2 new file mode 100644 index 0000000..3f02173 --- /dev/null +++ b/parcels/roles/components/clickhouse/common/templates/users.xml.j2 @@ -0,0 +1,202 @@ +<?xml version="1.0"?> +<yandex> + <!-- Profiles of settings. --> + <profiles> + <!-- Default settings. --> + <default> + <!-- Maximum memory usage for processing single query, in bytes. --> + <max_memory_usage>150000000000</max_memory_usage> + <max_memory_usage_for_all_queries>200000000000</max_memory_usage_for_all_queries> + <max_bytes_before_external_group_by>75000000000</max_bytes_before_external_group_by> + <distributed_aggregation_memory_efficient>1</distributed_aggregation_memory_efficient> + <distributed_product_mode>local</distributed_product_mode> + <log_queries>1</log_queries> + <cancel_http_readonly_queries_on_client_close>1</cancel_http_readonly_queries_on_client_close> + <background_pool_size>48</background_pool_size> + <!-- <enable_http_compression>1</enable_http_compression>--> + <replication_alter_columns_timeout>60</replication_alter_columns_timeout> + <skip_unavailable_shards>1</skip_unavailable_shards> + <max_execution_time>21600</max_execution_time> + <!-- Use cache of uncompressed blocks of data. Meaningful only for processing many of very short queries. --> + <use_uncompressed_cache>1</use_uncompressed_cache> + <replace_running_query>1</replace_running_query> + <http_receive_timeout>21600</http_receive_timeout> + <http_send_timeout>21600</http_send_timeout> + <receive_timeout>21600</receive_timeout> + <send_timeout>21600</send_timeout> + <count_distinct_implementation>uniqCombined</count_distinct_implementation> + <!-- How to choose between replicas during distributed query processing. + random - choose random replica from set of replicas with minimum number of errors + nearest_hostname - from set of replicas with minimum number of errors, choose replica + with minumum number of different symbols between replica's hostname and local hostname + (Hamming distance). + in_order - first live replica is choosen in specified order. + --> + <max_rows_to_group_by>10000000</max_rows_to_group_by> + <group_by_overflow_mode>any</group_by_overflow_mode> + <timeout_before_checking_execution_speed>3600</timeout_before_checking_execution_speed> + <load_balancing>in_order</load_balancing> + </default> + + <!-- Profile that allows only read queries. --> + <readonly> + <max_memory_usage>100000000000</max_memory_usage> + <max_memory_usage_for_all_queries>200000000000</max_memory_usage_for_all_queries> + <distributed_product_mode>local</distributed_product_mode> + <http_receive_timeout>600</http_receive_timeout> + <http_send_timeout>600</http_send_timeout> + <receive_timeout>600</receive_timeout> + <send_timeout>600</send_timeout> + <log_queries>1</log_queries> + <cancel_http_readonly_queries_on_client_close>1</cancel_http_readonly_queries_on_client_close> + <background_pool_size>48</background_pool_size> + <!-- http压缩 不影响http请求，只影响使用chproxy的客户端--> + <enable_http_compression>1</enable_http_compression> + <replace_running_query>1</replace_running_query> + <replication_alter_columns_timeout>60</replication_alter_columns_timeout> + <skip_unavailable_shards>1</skip_unavailable_shards> + <max_execution_time>600</max_execution_time> + <!-- Use cache of uncompressed blocks of data. Meaningful only for processing many of very short queries. --> + <timeout_before_checking_execution_speed>600</timeout_before_checking_execution_speed> + <use_uncompressed_cache>1</use_uncompressed_cache> + <count_distinct_implementation>uniqCombined</count_distinct_implementation> + <load_balancing>in_order</load_balancing> + <distributed_aggregation_memory_efficient>1</distributed_aggregation_memory_efficient> + <max_rows_to_group_by>10000000</max_rows_to_group_by> + <group_by_overflow_mode>any</group_by_overflow_mode> + <readonly>2</readonly> + + </readonly> + + <ckinsert> + <max_memory_usage>100000000000</max_memory_usage> + <max_memory_usage_for_all_queries>200000000000</max_memory_usage_for_all_queries> + <distributed_product_mode>local</distributed_product_mode> + <log_queries>1</log_queries> + <background_pool_size>48</background_pool_size> + + <replication_alter_columns_timeout>60</replication_alter_columns_timeout> + <skip_unavailable_shards>1</skip_unavailable_shards> + <max_execution_time>300</max_execution_time> + <!-- Use cache of uncompressed blocks of data. Meaningful only for processing many of very short queries. --> + <use_uncompressed_cache>0</use_uncompressed_cache> + <timeout_before_checking_execution_speed>300</timeout_before_checking_execution_speed> + <http_receive_timeout>300</http_receive_timeout> + <http_send_timeout>300</http_send_timeout> + <receive_timeout>300</receive_timeout> + <send_timeout>300</send_timeout> + <allow_ddl>0</allow_ddl> + <load_balancing>random</load_balancing> + </ckinsert> + </profiles> + + <!-- Users and ACL. --> + <users> + <!-- If user name was not specified, 'default' user is used. --> + <default> + <!-- Password could be specified in plaintext or in SHA256 (in hex format). + + If you want to specify password in plaintext (not recommended), place it in 'password' element. + Example: <password>qwerty</password>. + Password could be empty. + + If you want to specify SHA256, place it in 'password_sha256_hex' element. + Example: <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex> + + How to generate decent password: + Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-' + In first line will be password and in second - corresponding SHA256. + --> + <password_sha256_hex>{{ root_key_sha }}</password_sha256_hex> + + <!-- List of networks with open access. + + To open access from everywhere, specify: + <ip>::/0</ip> + + To open access only from localhost, specify: + <ip>::1</ip> + <ip>127.0.0.1</ip> + + Each element of list has one of the following forms: + <ip> IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0 + 2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::. + <host> Hostname. Example: server01.yandex.ru. + To check access, DNS query is performed, and all received addresses compared to peer address. + <host_regexp> Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$ + To check access, DNS PTR query is performed for peer address and then regexp is applied. + Then, for result of PTR query, another DNS query is performed and all received addresses compared to peer address. + Strongly recommended that regexp is ends with $ + All results of DNS requests are cached till server restart. + --> + <networks incl="networks" replace="replace"> + <ip>::/0</ip> + </networks> + + <!-- Settings profile for user. --> + <profile>default</profile> + + <!-- Quota for user. --> + <quota>default</quota> + </default> + + <tsg_report> + <password_sha256_hex>{{ root_key_sha }}</password_sha256_hex> + <networks incl="networks" replace="replace"> + <ip>::/0</ip> + </networks> + <profile>default</profile> + <quota>default</quota> + </tsg_report> + + + <tsg_insert> + <password_sha256_hex>{{ root_key_sha }}</password_sha256_hex> + <networks incl="networks" replace="replace"> + <ip>::/0</ip> + </networks> + <profile>ckinsert</profile> + <quota>default</quota> + </tsg_insert> + + <!-- Example of user with readonly access. --> + <tsg_query> + <password_sha256_hex>{{ readonly_key_sha }}</password_sha256_hex> + <networks incl="networks" replace="replace"> + <ip>::/0</ip> + </networks> + <profile>readonly</profile> + <quota>default</quota> + </tsg_query> + + + <!-- Example of user with readonly access. --> + <readonly> + <password></password> + <networks incl="networks" replace="replace"> + <ip>::1</ip> + <ip>127.0.0.1</ip> + </networks> + <profile>readonly</profile> + <quota>default</quota> + </readonly> + </users> + + <!-- Quotas. --> + <quotas> + <!-- Name of quota. --> + <default> + <!-- Limits for time interval. You could specify many intervals with different limits. --> + <interval> + <!-- Length of interval. --> + <duration>3600</duration> + <!-- No limits. Just calculate resource usage for time interval. --> + <queries>0</queries> + <errors>0</errors> + <result_rows>0</result_rows> + <read_rows>0</read_rows> + <execution_time>0</execution_time> + </interval> + </default> + </quotas> +</yandex> diff --git a/parcels/roles/components/clickhouse/common/vars/main.yml b/parcels/roles/components/clickhouse/common/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/components/clickhouse/common/vars/main.yml diff --git a/parcels/roles/components/clickhouse/data/tasks/install_clickhouse.yml b/parcels/roles/components/clickhouse/data/tasks/install_clickhouse.yml new file mode 100644 index 0000000..2571627 --- /dev/null +++ b/parcels/roles/components/clickhouse/data/tasks/install_clickhouse.yml @@ -0,0 +1,54 @@ +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181" + when: '(groups.zookeeper|length) == 3' + run_once: true + delegate_to: 127.0.0.1 + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181,{{groups.zookeeper[3]}}:2181,{{groups.zookeeper[4]}}:2181" + when: '(groups.zookeeper|length) >= 5' + run_once: true + delegate_to: 127.0.0.1 + +#拷贝clickhouse 配置文件 +- name: copy clickhouse data config files + template: + src: 'ck_cluster.xml.j2' + dest: '{{ data_path }}/clickhouse/clickhouse-server/metrika.xml' + force: true + backup: yes + when: inventory_hostname != (groups.clickhouse[0]) and inventory_hostname != (groups.clickhouse[1]) + +- name: check unixODBC + shell: rpm -qa | grep "unixODBC-2.3.1-11.el7.x86_64" | wc -l + register: odbc_install +# when: inventory_hostname != (groups.clickhouse[0]) and inventory_hostname != (groups.clickhouse[1]) + +- name: set zookeeper servers to clickhouse config + shell: cd {{ data_path }}/clickhouse/clickhouse-server/ && ./set_zk_node.sh + when: inventory_hostname != (groups.clickhouse[0]) and inventory_hostname != (groups.clickhouse[1]) + +- name: install clickhouse + shell: "if [ `rpm -qa | grep {{ item.file }} | wc -l` -eq '0' ]; then rpm -ivh {{ data_path }}/clickhouse/rpm/{{ item.file }}.rpm ; fi" + with_items: + - { file: 'clickhouse-common-static-{{ clickhouse_version }}-1.el7.x86_64' } + - { file: 'clickhouse-server-common-{{ clickhouse_version }}-1.el7.x86_64' } + - { file: 'clickhouse-server-{{ clickhouse_version }}-1.el7.x86_64' } + - { file: 'clickhouse-client-{{ clickhouse_version }}-1.el7.x86_64' } + when: odbc_install.stdout != '0' and inventory_hostname != (groups.clickhouse[0]) and inventory_hostname != (groups.clickhouse[1]) + +- name: set dir user + shell: chown clickhouse:clickhouse -R {{ data_path }}/clickhouse/ + when: inventory_hostname != (groups.clickhouse[0]) and inventory_hostname != (groups.clickhouse[1]) + +- name: copy clickhouse-server to /etc/init.d/ + shell: yes | cp -r {{ data_path }}/clickhouse/clickhouse-server/clickhouse-server /etc/init.d/clickhouse-server + when: inventory_hostname != (groups.clickhouse[0]) and inventory_hostname != (groups.clickhouse[1]) + +- name: deleted tmp file + shell: rm -rf {{ data_path }}/clickhouse/clickhouse-server/ckquery_list {{ data_path }}/clickhouse/clickhouse-server/ckdata_list {{ data_path }}/clickhouse/clickhouse-server/set_zk_node.sh + when: inventory_hostname != (groups.clickhouse[0]) and inventory_hostname != (groups.clickhouse[1]) + +- name: start clickhouse + shell: service clickhouse-server start + when: inventory_hostname != (groups.clickhouse[0]) and inventory_hostname != (groups.clickhouse[1]) diff --git a/parcels/roles/components/clickhouse/data/tasks/main.yml b/parcels/roles/components/clickhouse/data/tasks/main.yml new file mode 100644 index 0000000..dceb53f --- /dev/null +++ b/parcels/roles/components/clickhouse/data/tasks/main.yml @@ -0,0 +1 @@ +- include: install_clickhouse.yml diff --git a/parcels/roles/components/clickhouse/data/templates/ck_cluster.xml.j2 b/parcels/roles/components/clickhouse/data/templates/ck_cluster.xml.j2 new file mode 100644 index 0000000..a3ecc56 --- /dev/null +++ b/parcels/roles/components/clickhouse/data/templates/ck_cluster.xml.j2 @@ -0,0 +1,40 @@ +<yandex> +<!--ck集群节点--> +<clickhouse_remote_servers> + +<ck_cluster> + <shard> + <!-- Optional. Shard weight when writing data. Default: 1. --> + <weight>1</weight> + <!-- Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). --> + <internal_replication>false</internal_replication> + <replica> + <host>{{ inventory_hostname }}</host> + <port>9001</port> + <user>default</user> + <password>{{ root_key }}</password> + </replica> + </shard> +</ck_cluster> + +</clickhouse_remote_servers> +<zookeeper-servers> +<session_timeout_ms>120000</session_timeout_ms> +</zookeeper-servers> + +<networks> +<ip>::/0</ip> +</networks> + +<!--压缩相关配置--> +<clickhouse_compression> +<case> +<min_part_size>10000000000</min_part_size> +<min_part_size_ratio>0.01</min_part_size_ratio> +<method>lz4</method> <!--压缩算法lz4压缩比zstd快, 更占磁盘--> +</case> +</clickhouse_compression> +</yandex> + + + diff --git a/parcels/roles/components/clickhouse/data/vars/main.yml b/parcels/roles/components/clickhouse/data/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/components/clickhouse/data/vars/main.yml diff --git a/parcels/roles/components/clickhouse/query/tasks/install_clickhouse.yml b/parcels/roles/components/clickhouse/query/tasks/install_clickhouse.yml new file mode 100644 index 0000000..6f3ec2f --- /dev/null +++ b/parcels/roles/components/clickhouse/query/tasks/install_clickhouse.yml @@ -0,0 +1,125 @@ +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181" + when: '(groups.zookeeper|length) == 3' + run_once: true + delegate_to: 127.0.0.1 + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181,{{groups.zookeeper[3]}}:2181,{{groups.zookeeper[4]}}:2181" + when: '(groups.zookeeper|length) >= 5' + run_once: true + delegate_to: 127.0.0.1 + +#拷贝clickhouse 配置文件 +- name: copy clickhouse config files + template: + src: 'ck_query.xml.j2' + dest: '{{ data_path }}/clickhouse/clickhouse-server/metrika.xml' + force: true + backup: yes + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.clickhouse[0] }}' } + - { ip: '{{ groups.clickhouse[1] }}' } + +- name: copy set_query_node.sh to {{ data_path }}/clickhouse/clickhouse-server/ + template: + src: 'set_query_node.sh.j2' + dest: '{{ data_path }}/clickhouse/clickhouse-server/set_query_node.sh' + force: true + mode: 0755 + backup: yes + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.clickhouse[0] }}' } + - { ip: '{{ groups.clickhouse[1] }}' } + +- name: check unixODBC + shell: rpm -qa | grep "unixODBC-2.3.1-11.el7.x86_64" | wc -l + register: odbc_install + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.clickhouse[0] }}' } + - { ip: '{{ groups.clickhouse[1] }}' } + +- name: set zookeeper servers to clickhouse config + shell: cd {{ data_path }}/clickhouse/clickhouse-server/ && ./set_zk_node.sh + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.clickhouse[0] }}' } + - { ip: '{{ groups.clickhouse[1] }}' } + +- name: set clickhouse query config + shell: cd {{ data_path }}/clickhouse/clickhouse-server/ && ./set_query_node.sh + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.clickhouse[0] }}' } + - { ip: '{{ groups.clickhouse[1] }}' } + +- name: install clickhouse + shell: "if [ `rpm -qa | grep {{ item.file }} | wc -l` -eq '0' ]; then rpm -ivh {{ data_path }}/clickhouse/rpm/{{ item.file }}.rpm ; fi" + with_items: + - { file: 'clickhouse-common-static-{{ clickhouse_version }}-1.el7.x86_64' } + - { file: 'clickhouse-server-common-{{ clickhouse_version }}-1.el7.x86_64' } + - { file: 'clickhouse-server-{{ clickhouse_version }}-1.el7.x86_64' } + - { file: 'clickhouse-client-{{ clickhouse_version }}-1.el7.x86_64' } + run_once: true + delegate_facts: true + delegate_to: "{{ groups.clickhouse[0] }}" + +- name: install clickhouse + shell: "if [ `rpm -qa | grep {{ item.file }} | wc -l` -eq '0' ]; then rpm -ivh {{ data_path }}/clickhouse/rpm/{{ item.file }}.rpm ; fi" + with_items: + - { file: 'clickhouse-common-static-{{ clickhouse_version }}-1.el7.x86_64' } + - { file: 'clickhouse-server-common-{{ clickhouse_version }}-1.el7.x86_64' } + - { file: 'clickhouse-server-{{ clickhouse_version }}-1.el7.x86_64' } + - { file: 'clickhouse-client-{{ clickhouse_version }}-1.el7.x86_64' } + run_once: true + delegate_facts: true + delegate_to: "{{ groups.clickhouse[1] }}" + +- name: set dir user + shell: chown clickhouse:clickhouse -R {{ data_path }}/clickhouse/ + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.clickhouse[0] }}' } + - { ip: '{{ groups.clickhouse[1] }}' } + +- name: copy clickhouse-server to /etc/init.d/ + shell: yes | cp -r {{ data_path }}/clickhouse/clickhouse-server/clickhouse-server /etc/init.d/clickhouse-server + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.clickhouse[0] }}' } + - { ip: '{{ groups.clickhouse[1] }}' } + +- name: deleted tmp file + shell: rm -rf {{ data_path }}/clickhouse/clickhouse-server/ckquery_list {{ data_path }}/clickhouse/clickhouse-server/ckdata_list {{ data_path }}/clickhouse/clickhouse-server/set_zk_node.sh {{ data_path }}/clickhouse/clickhouse-server/set_query_node.sh + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.clickhouse[0] }}' } + - { ip: '{{ groups.clickhouse[1] }}' } + +- name: start clickhouse + shell: service clickhouse-server start + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.clickhouse[0] }}' } + - { ip: '{{ groups.clickhouse[1] }}' } diff --git a/parcels/roles/components/clickhouse/query/tasks/main.yml b/parcels/roles/components/clickhouse/query/tasks/main.yml new file mode 100644 index 0000000..dceb53f --- /dev/null +++ b/parcels/roles/components/clickhouse/query/tasks/main.yml @@ -0,0 +1 @@ +- include: install_clickhouse.yml diff --git a/parcels/roles/components/clickhouse/query/templates/ck_query.xml.j2 b/parcels/roles/components/clickhouse/query/templates/ck_query.xml.j2 new file mode 100644 index 0000000..974c652 --- /dev/null +++ b/parcels/roles/components/clickhouse/query/templates/ck_query.xml.j2 @@ -0,0 +1,42 @@ +<yandex> +<!--ck集群节点--> +<clickhouse_remote_servers> + +<!--clickhouse query集群节点--> +<ck_query> + +</ck_query> + + +<!--clickhouse cluster集群节点--> +<ck_cluster> + +</ck_cluster> + + +<!--clickhouse 所有节点--> +<ck_all> + +</ck_all> + +</clickhouse_remote_servers> +<zookeeper-servers> +<session_timeout_ms>120000</session_timeout_ms> +</zookeeper-servers> + +<networks> +<ip>::/0</ip> +</networks> + +<!--压缩相关配置--> +<clickhouse_compression> +<case> +<min_part_size>10000000000</min_part_size> +<min_part_size_ratio>0.01</min_part_size_ratio> +<method>lz4</method> <!--压缩算法lz4压缩比zstd快, 更占磁盘--> +</case> +</clickhouse_compression> +</yandex> + + + diff --git a/parcels/roles/components/clickhouse/query/templates/set_query_node.sh.j2 b/parcels/roles/components/clickhouse/query/templates/set_query_node.sh.j2 new file mode 100755 index 0000000..a2a52b7 --- /dev/null +++ b/parcels/roles/components/clickhouse/query/templates/set_query_node.sh.j2 @@ -0,0 +1,34 @@ +#!/bin/bash + +# <shard> +# <!-- Optional. Shard weight when writing data. Default: 1. --> +# <weight>1</weight> +# <!-- Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). --> +# <internal_replication>false</internal_replication> +# <replica> +# <host>host</host> +# <port>9001</port> +# <user>default</user> +# <password>password</password> +# </replica> +# </shard> + +function set_conf(){ +line_num=`cat -n {{ data_path }}/clickhouse/clickhouse-server/metrika.xml |grep $1 |awk '{print $1}'|sed -n "1"p` +line_num=`expr $line_num + 1` +for node in `cat $2` +do + ipport=(${node//:/ }) + sed -i ''$line_num'a<shard>\n <!-- Optional. Shard weight when writing data. Default: 1. -->\n <weight>1</weight>\n <!-- Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). -->\n <internal_replication>false</internal_replication>\n <replica>\n <host>'$node'</host>\n <port>9001</port>\n <user>default</user>\n <password>{{ root_key }}</password>\n </replica>\n</shard>\n' {{ data_path }}/clickhouse/clickhouse-server/metrika.xml + start_num=`expr $line_num + 13` +done + +} + +set_conf ck_query ckquery_list + +set_conf ck_cluster ckdata_list + +set_conf ck_all ckdata_list + +set_conf ck_all ckquery_list diff --git a/parcels/roles/components/clickhouse/query/vars/main.yml b/parcels/roles/components/clickhouse/query/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/components/clickhouse/query/vars/main.yml diff --git a/parcels/roles/components/druid/files/druid_ips b/parcels/roles/components/druid/files/druid_ips new file mode 100644 index 0000000..9618e84 --- /dev/null +++ b/parcels/roles/components/druid/files/druid_ips @@ -0,0 +1,3 @@ +192.168.44.33 +192.168.44.34 +192.168.44.35 diff --git a/parcels/roles/components/druid/files/mysql b/parcels/roles/components/druid/files/mysql Binary files differnew file mode 100755 index 0000000..66af1a1 --- /dev/null +++ b/parcels/roles/components/druid/files/mysql diff --git a/parcels/roles/components/druid/tasks/install_druid.yml b/parcels/roles/components/druid/tasks/install_druid.yml new file mode 100644 index 0000000..f3cd61e --- /dev/null +++ b/parcels/roles/components/druid/tasks/install_druid.yml @@ -0,0 +1,278 @@ +- name: node_nums to ansible variable + set_fact: node_nums="{{groups.druid|length}}" + +- name: To terminate execution + fail: + msg: "Druid 集群最少需要配置3个节点，请修改configurations/hosts内对应IP列表。" + when: node_nums < '3' + +- name: check jdk 1.8_73 + shell: source /etc/profile && java -version 2>&1 | grep 1.8.0_73 | wc -l + ignore_errors: false + register: jdk_out + +- name: To terminate execution + fail: + msg: "检测到目标集群内存在JDK未安装节点，请检查" + when: jdk_out.stdout != '2' + run_once: true + delegate_to: 127.0.0.1 + +- name: copy mysql to /usr/bin/ + copy: + src: '../files/mysql' + dest: '/usr/bin/' + force: true + +- name: change mode + shell: chmod +x /usr/bin/mysql + +- block: + - name: Check if the Druid service already exists + shell: ps -ef |grep "org.apache.druid.cli.Main server" | grep -v grep | grep -v json | wc -l + register: check_out + - name: copy unload_druid.sh to {{ install_path }}/ + template: + src: 'unload_druid.sh.j2' + dest: '{{ install_path }}/unload_druid.sh' + force: true + mode: 0755 + when: check_out.stdout >= '1' + - name: unload kafka + shell: cd {{ install_path }} && sh unload_druid.sh + when: check_out.stdout >= '1' + - name: check if {{ mariadb_druid_database }} exist + shell: mysql -s -uroot -p{{ galaxy_mariadb_pin }} -P3306 -h{{ gateway_keepalive_host }} -e "select count(1) from information_schema.schemata where schema_name='{{ mariadb_druid_database }}'" |grep 1|wc -l + register: has_druid_out + run_once: true + delegate_facts: true + delegate_to: "127.0.0.1" + - name: check database + shell: mysql -s -uroot -p{{ galaxy_mariadb_pin }} -P3306 -h{{ gateway_keepalive_host }} -e "drop database {{ mariadb_druid_database }};" + run_once: true + delegate_facts: true + delegate_to: "127.0.0.1" + when: has_druid_out.stdout == '1' + - name: Ansible delete {{ install_path }}/unload_druid.sh + file: + path: "{{ install_path }}/unload_druid.sh" + state: absent + - name: Check if the Druid service already exists + shell: ps -ef |grep "org.apache.druid.cli.Main server" | grep -v grep | grep -v json | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "卸载失败，组件可能非本安装部署，请手动卸载后继续安装" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "yes" + +- block: + - name: Check if the Druid service already exists + shell: ps -ef |grep "org.apache.druid.cli.Main server" | grep -v grep | grep -v json | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "检测到目标集群内已有Druid请检查IP是否正确或停止/卸载已启动的组件" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "no" + +- name: hadoop_ip to ansible variable + set_fact: hadoop_ip="{{groups.hadoop[0]}}" + +#- name: check if {{ mariadb_druid_database }} exist +# shell: mysql -s -uroot -p{{ galaxy_mariadb_pin }} -P3306 -h{{ gateway_keepalive_host }} -e "select count(1) from information_schema.schemata where schema_name='{{ mariadb_druid_database }}'" |grep 1|wc -l +# register: has_druid_out +# run_once: true +# delegate_facts: true +# delegate_to: "127.0.0.1" + +#- name: check database +# fail: +# msg: "mariadb已经存在Druid数据库，请确认删除后再次执行安装" +# when: has_druid_out.stdout == '1' + +- name: check hadoop datanode num ,If the service cannot ,Time out after 60 seconds + shell: source /etc/profile && hadoop dfsadmin -report | grep "Live datanodes" | grep -E -o "[0-9]" + async: 60 + register: datanode_out + run_once: true + delegate_facts: true + delegate_to: "{{ hadoop_ip }}" + +- name: check hadoop namenode num ,If the service cannot ,Time out after 60 seconds + shell: source /etc/profile && hadoop dfsadmin -report |grep 50010 | wc -l + async: 60 + register: namenode_out + run_once: true + delegate_facts: true + delegate_to: "{{ hadoop_ip }}" + +- fail: + msg: "Hadoop 服务节点启动检测异常，请检查服务或是否已做免密登录" + when: datanode_out.stdout <= '1' and namenode_out.stdout <= '1' + +- name: create druid package path:{{ install_path }} + file: + state: directory + path: '{{ install_path }}' + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181" + when: '(groups.zookeeper|length) == 3' + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181,{{groups.zookeeper[3]}}:2181,{{groups.zookeeper[4]}}:2181" + when: '(groups.zookeeper|length) >= 5' + +- name: Create a new database with name {{ mariadb_druid_database }} + shell: mysql -uroot -p{{ galaxy_mariadb_pin }} -P3306 -h{{ gateway_keepalive_host }} -e "create database {{ mariadb_druid_database }} default character set utf8mb4 collate utf8mb4_general_ci;" + run_once: true + delegate_facts: true + delegate_to: "127.0.0.1" + +#复制tar到目标服务器 +- name: copy {{ druid_version }}.tar.gz + copy: + src: '{{ package_path }}/{{ druid_version }}.tar.gz' + dest: '{{ install_path }}/' + force: true + backup: yes + +#解压tar +- name: unpack {{ druid_version }}.tar.gz + unarchive: + src: '{{ druid_version }}.tar.gz' + dest: '{{ install_path }}/' + +#拷贝druid 配置文件 +- name: copy druid config files + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + backup: yes + with_items: + - { src: 'common.runtime.properties.j2', dest: '{{ install_path }}/{{ druid_version }}/conf/druid/cluster/_common/common.runtime.properties' } + - { src: 'log4j2.xml.j2', dest: '{{ install_path }}/{{ druid_version }}/conf/druid/cluster/_common/log4j2.xml' } + - { src: 'broker_runtime.j2', dest: '{{ install_path }}/{{ druid_version }}/conf/druid/cluster/query/broker/runtime.properties' } + - { src: 'historical_runtime.j2', dest: '{{ install_path }}/{{ druid_version }}/conf/druid/cluster/data/historical/runtime.properties' } + - { src: 'historical_jvm.j2', dest: '{{ install_path }}/{{ druid_version }}/conf/druid/cluster/data/historical/jvm.config' } + - { src: 'middleManager_runtime.properties.j2', dest: '{{ install_path }}/{{ druid_version }}/conf/druid/cluster/data/middleManager/runtime.properties' } + - { src: 'coordinator_jvm.j2', dest: '{{ install_path }}/{{ druid_version }}/conf/druid/cluster/master/coordinator-overlord/jvm.config' } + - { src: 'broker_jvm.j2', dest: '{{ install_path }}/{{ druid_version }}/conf/druid/cluster/query/broker/jvm.config' } + +#拷贝hadoop配置文件到 druid/conf +- name: copy hadoop-conf to {{ install_path }}/{{ druid_version }}/conf/druid/cluster/_common/ + shell: "scp root@{{ hadoop_ip }}:{{ data_path }}/hadoop-2.7.1/etc/hadoop/{{ item.filename }} {{ install_path }}/{{ druid_version }}/conf/druid/cluster/_common/" + with_items: + - { filename: 'hdfs-site.xml' } + - { filename: 'core-site.xml' } + - { filename: 'yarn-site.xml' } + - { filename: 'mapred-site.xml.bak' } + +- name: master_ip to ansible variable + set_fact: master_ip="{{groups.druid[0]}}" + +- name: slave1_ip to ansible variable + set_fact: slave1_ip="{{groups.druid[1]}}" + +#拷贝druid 守护脚本/开机自启 +- name: copy Druid all start guardian/keep shell + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + mode: 0755 + backup: yes + with_items: + - { src: 'dae-druid-all.sh.j2', dest: '{{ install_path }}/{{ druid_version }}/bin/dae-druid-all.sh' } + - { src: 'keepdruidall.j2', dest: '/etc/init.d/keepdruidall' } + when: node_nums <= cluster_limit + +#拷贝druid 配置文件 +- name: copy Druid query node guardian/keep shell for {{ master_ip }} + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + mode: 0755 + backup: yes + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + with_items: + - { src: 'dae-druid-query.sh.j2', dest: '{{ install_path }}/{{ druid_version }}/bin/dae-druid-query.sh' } + - { src: 'keepdruidquery.j2', dest: '/etc/init.d/keepdruidquery' } + when: node_nums > cluster_limit + +#拷贝druid 配置文件 +- name: copy Druid query node guardian/keep shell for {{ slave1_ip }} + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + mode: 0755 + backup: yes + run_once: true + delegate_facts: true + delegate_to: "{{ slave1_ip }}" + with_items: + - { src: 'dae-druid-query.sh.j2', dest: '{{ install_path }}/{{ druid_version }}/bin/dae-druid-query.sh' } + - { src: 'keepdruidquery.j2', dest: '/etc/init.d/keepdruidquery' } + when: node_nums > cluster_limit + +#拷贝druid 配置文件 +- name: copy Druid data node guardian shell + template: + src: 'dae-druid-data.sh.j2' + dest: '{{ install_path }}/{{ druid_version }}/bin/dae-druid-data.sh' + mode: 0755 + backup: yes + when: node_nums > cluster_limit and inventory_hostname != '{{master_ip}}' and inventory_hostname != '{{slave1_ip}}' + +#拷贝druid 配置文件 +- name: copy Druid data node keep shell + template: + src: 'keepdruiddata.j2' + dest: '/etc/init.d/keepdruiddata' + mode: 0755 + backup: yes + when: node_nums > cluster_limit and inventory_hostname != '{{master_ip}}' and inventory_hostname != '{{slave1_ip}}' + +#拷贝set_druid_env.sh +- name: copy set_druid_env.sh + template: + src: 'set_druid_env.sh.j2' + dest: '{{ install_path }}/{{ druid_version }}/bin/set_druid_env.sh' + mode: 0755 + backup: yes + +#配置druid环境变量 +- name: set druid env + shell: sh {{ install_path }}/{{ druid_version }}/bin/set_druid_env.sh + +#拷贝druid 配置文件 +- name: start druid query node + shell: source /etc/profile && sh /etc/init.d/keepdruidquery 'start' + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ master_ip }}' } + - { ip: '{{ slave1_ip }}' } + when: node_nums > cluster_limit + +#拷贝druid 配置文件 +- name: start druid data node + shell: source /etc/profile && sh /etc/init.d/keepdruiddata 'start' + when: node_nums > cluster_limit and inventory_hostname != '{{master_ip}}' and inventory_hostname != '{{slave1_ip}}' + +#拷贝druid 配置文件 +- name: start druid all node + shell: source /etc/profile && sh /etc/init.d/keepdruidall 'start' + when: node_nums <= cluster_limit + +- name: Ansible delete {{ druid_version }}.tar.gz + file: + path: "{{ install_path }}/{{ druid_version }}.tar.gz" + state: absent diff --git a/parcels/roles/components/druid/tasks/main.yml b/parcels/roles/components/druid/tasks/main.yml new file mode 100644 index 0000000..dd24c02 --- /dev/null +++ b/parcels/roles/components/druid/tasks/main.yml @@ -0,0 +1 @@ +- include: install_druid.yml diff --git a/parcels/roles/components/druid/templates/broker_jvm.j2 b/parcels/roles/components/druid/templates/broker_jvm.j2 new file mode 100644 index 0000000..504dc28 --- /dev/null +++ b/parcels/roles/components/druid/templates/broker_jvm.j2 @@ -0,0 +1,10 @@ +-server +-Xms{{ broker_mem }}m +-Xmx{{ broker_mem }}m +-XX:MaxDirectMemorySize={{ broker_MaxDirectMemorySize }}m +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Dlogfile.name=broker +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager + diff --git a/parcels/roles/components/druid/templates/broker_runtime.j2 b/parcels/roles/components/druid/templates/broker_runtime.j2 new file mode 100644 index 0000000..9d0a470 --- /dev/null +++ b/parcels/roles/components/druid/templates/broker_runtime.j2 @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/broker +druid.plaintextPort=8082 +druid.sql.planner.sqlTimeZone={{ local_time }} + +# HTTP server settings +druid.server.http.numThreads=60 + +# HTTP client settings +druid.broker.http.numConnections=50 +druid.broker.http.maxQueuedBytes=10000000 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes={{ broker_sizeBytes }} +druid.processing.numMergeBuffers={{ broker_numMergeBuffers }} +druid.processing.numThreads={{ broker_numThreads }} +druid.processing.tmpDir=var/druid/processing + +# Query cache disabled -- push down caching and merging instead +druid.broker.cache.useCache=false +druid.broker.cache.populateCache=false + +druid.query.groupBy.maxMergingDictionarySize=10000000000 +druid.query.groupBy.maxOnDiskStorage=10000000000 diff --git a/parcels/roles/components/druid/templates/common.runtime.properties.j2 b/parcels/roles/components/druid/templates/common.runtime.properties.j2 new file mode 100644 index 0000000..ef8cce3 --- /dev/null +++ b/parcels/roles/components/druid/templates/common.runtime.properties.j2 @@ -0,0 +1,151 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Extensions specified in the load list will be loaded by Druid +# We are using local fs for deep storage - not recommended for production - use S3, HDFS, or NFS instead +# We are using local derby for the metadata store - not recommended for production - use MySQL or Postgres instead + +# If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. +# If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. +# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html +druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "mysql-metadata-storage", "druid-group-uniq-extension"] + +# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory +# and uncomment the line below to point to your directory. +#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies + + +# +# Hostname +# +druid.host={{ inventory_hostname }} + +# +# Logging +# + +# Log all runtime properties on startup. Disable to avoid logging properties on startup: +druid.startup.logging.logProperties=true + +# +# Zookeeper +# + +druid.zk.service.host={{ zookeeper_servers }} +druid.zk.paths.base=/druid + +# +# Metadata storage +# + +# For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): +#druid.metadata.storage.type=derby +#druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true +#druid.metadata.storage.connector.host=localhost +#druid.metadata.storage.connector.port=1527 + +# For MySQL (make sure to include the MySQL JDBC driver on the classpath): +druid.metadata.storage.type=mysql +druid.metadata.storage.connector.connectURI=jdbc:mysql://{{ gateway_keepalive_host }}:3306/{{ mariadb_druid_database }} +druid.metadata.storage.connector.user=root +druid.metadata.storage.connector.password={{ galaxy_mariadb_pin }} + +# For PostgreSQL: +#druid.metadata.storage.type=postgresql +#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid +#druid.metadata.storage.connector.user=... +#druid.metadata.storage.connector.password=... + +# +# Deep storage +# + +# For local disk (only viable in a cluster if this is a network mount): +#druid.storage.type=local +#druid.storage.storageDirectory=var/druid/segments + +# For HDFS: +druid.storage.type=hdfs +druid.storage.storageDirectory=/druid/segments + +# For S3: +#druid.storage.type=s3 +#druid.storage.bucket=your-bucket +#druid.storage.baseKey=druid/segments +#druid.s3.accessKey=... +#druid.s3.secretKey=... + +# +# Indexing service logs +# + +# For local disk (only viable in a cluster if this is a network mount): +#druid.indexer.logs.type=file +#druid.indexer.logs.directory=var/druid/indexing-logs + +# For HDFS: +druid.indexer.logs.type=hdfs +druid.indexer.logs.directory=/druid/indexing-logs + +druid.indexer.logs.kill.enabled=true +druid.indexer.logs.kill.durationToRetain=604800000 +druid.indexer.logs.kill.delay=21600000 + +# For S3: +#druid.indexer.logs.type=s3 +#druid.indexer.logs.s3Bucket=your-bucket +#druid.indexer.logs.s3Prefix=druid/indexing-logs + + +# +# Service discovery +# + +druid.selectors.indexing.serviceName=druid/overlord +druid.selectors.coordinator.serviceName=druid/coordinator + +# +# Monitoring +# + +druid.monitoring.monitors=["org.apache.druid.java.util.metrics.SysMonitor","org.apache.druid.java.util.metrics.JvmMonitor"] +druid.emitter=http +druid.emitter.logging.logLevel=info +druid.emitter.http.recipientBaseUrl=http://{{ inventory_hostname }}:9903 + +# Storage type of double columns +# ommiting this will lead to index double as float at the storage layer + +druid.indexing.doubleStorage=double + +# +# Security +# +druid.server.hiddenProperties=["druid.s3.accessKey","druid.s3.secretKey","druid.metadata.storage.connector.password"] + + +# +# SQL +# +druid.sql.enable=true + +# +# Lookups +# +druid.lookup.enableLookupSyncOnStartup=false diff --git a/parcels/roles/components/druid/templates/coordinator_jvm.j2 b/parcels/roles/components/druid/templates/coordinator_jvm.j2 new file mode 100644 index 0000000..d7d4e17 --- /dev/null +++ b/parcels/roles/components/druid/templates/coordinator_jvm.j2 @@ -0,0 +1,11 @@ +-server +-Xms{{ coordinator_mem }}m +-Xmx{{ coordinator_mem }}m +-XX:+UseG1GC +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Dlogfile.name=coordinator +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +-Dderby.stream.error.file=var/druid/derby.log + diff --git a/parcels/roles/components/druid/templates/dae-druid-all.sh.j2 b/parcels/roles/components/druid/templates/dae-druid-all.sh.j2 new file mode 100644 index 0000000..d53ee10 --- /dev/null +++ b/parcels/roles/components/druid/templates/dae-druid-all.sh.j2 @@ -0,0 +1,31 @@ +#!/bin/bash +source /etc/profile + +BASH_DIR={{ install_path }} +VERSION={{ druid_version }} + +while true ; do + + metrics=`ps -ef | grep druid-metrics | grep -v grep | wc -l` + druid=`ps -ef | grep druid | grep -v grep | grep -v json | grep cluster.conf| wc -l` + + if [ $druid -eq "0" ];then + setsid nohup $BASH_DIR/$VERSION/bin/start-cluster-all-server > /dev/null 2>&1 & + OLD_NUM=`cat $BASH_DIR/$VERSION/protect/restartsum/all-server` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASh_DIR/$VERSION/protect/restartsum/all-server + echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart start-cluster-all-server - 重启次数 -> $RESTART_NUM." >> $BASH_DIR/$VERSION/protect/protecthdfs.log + fi + + if [ $metrics -eq "0" ];then + nohup java -jar -Xmx1024m -Xms1024m $BASH_DIR/$VERSION/monitor/druid-metrics.jar 9903 > $BASH_DIR/$VERSION/monitor/metrics.log 2>&1 & + OLD_NUM=`cat $BASH_DIR/$VERSION/protect/restartsum/metrics` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASH_DIR/$VERSION/protect/restartsum/metrics + echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart druid-metrics.jar - 重启次数 -> $RESTART_NUM." >> $BASH_DIR/$VERSION/protect/protecthdfs.log + fi + + +sleep 60 +done + diff --git a/parcels/roles/components/druid/templates/dae-druid-data.sh.j2 b/parcels/roles/components/druid/templates/dae-druid-data.sh.j2 new file mode 100644 index 0000000..c7266c2 --- /dev/null +++ b/parcels/roles/components/druid/templates/dae-druid-data.sh.j2 @@ -0,0 +1,31 @@ +#!/bin/bash +source /etc/profile + +BASH_DIR={{ install_path }} +VERSION={{ druid_version }} + +while true ; do + + metrics=`ps -ef | grep druid-metrics | grep -v grep | wc -l` + druid=`ps -ef | grep druid | grep -v grep | grep -v json | grep data.conf | wc -l` + + if [ $druid -eq "0" ];then + setsid nohup $BASH_DIR/$VERSION/bin/start-cluster-data-server > /dev/null 2>&1 & + OLD_NUM=`cat $BASH_DIR/$VERSION/protect/restartsum/data-server` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASh_DIR/$VERSION/protect/restartsum/data-server + echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart start-cluster-data-server - 重启次数 -> $RESTART_NUM." >> $BASH_DIR/$VERSION/protect/protecthdfs.log + fi + + if [ $metrics -eq "0" ];then + nohup java -jar -Xmx1024m -Xms1024m $BASH_DIR/$VERSION/monitor/druid-metrics.jar 9903 > $BASH_DIR/$VERSION/monitor/metrics.log 2>&1 & + OLD_NUM=`cat $BASH_DIR/$VERSION/protect/restartsum/metrics` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASH_DIR/$VERSION/protect/restartsum/metrics + echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart druid-metrics.jar - 重启次数 -> $RESTART_NUM." >> $BASH_DIR/$VERSION/protect/protecthdfs.log + fi + + +sleep 60 +done + diff --git a/parcels/roles/components/druid/templates/dae-druid-query.sh.j2 b/parcels/roles/components/druid/templates/dae-druid-query.sh.j2 new file mode 100644 index 0000000..3c3bd2a --- /dev/null +++ b/parcels/roles/components/druid/templates/dae-druid-query.sh.j2 @@ -0,0 +1,31 @@ +#!/bin/bash +source /etc/profile + +BASH_DIR={{ install_path }} +VERSION={{ druid_version }} + +while true ; do + + metrics=`ps -ef | grep druid-metrics | grep -v grep | wc -l` + druid=`ps -ef | grep druid | grep -v grep | grep -v json | grep master-with-query.conf | wc -l` + + if [ $druid -eq "0" ];then + setsid nohup $BASH_DIR/$VERSION/bin/start-cluster-query-server > /dev/null 2>&1 & + OLD_NUM=`cat $BASH_DIR/$VERSION/protect/restartsum/query-server` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASh_DIR/$VERSION/protect/restartsum/query-server + echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart start-cluster-query-server - 重启次数 -> $RESTART_NUM." >> $BASH_DIR/$VERSION/protect/protecthdfs.log + fi + + if [ $metrics -eq "0" ];then + nohup java -jar -Xmx1024m -Xms1024m $BASH_DIR/$VERSION/monitor/druid-metrics.jar 9903 > $BASH_DIR/$VERSION/monitor/metrics.log 2>&1 & + OLD_NUM=`cat $BASH_DIR/$VERSION/protect/restartsum/metrics` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASH_DIR/$VERSION/protect/restartsum/metrics + echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart druid-metrics.jar - 重启次数 -> $RESTART_NUM." >> $BASH_DIR/$VERSION/protect/protecthdfs.log + fi + + +sleep 60 +done + diff --git a/parcels/roles/components/druid/templates/historical_jvm.j2 b/parcels/roles/components/druid/templates/historical_jvm.j2 new file mode 100644 index 0000000..6dd1083 --- /dev/null +++ b/parcels/roles/components/druid/templates/historical_jvm.j2 @@ -0,0 +1,10 @@ +-server +-Xms{{ historical_mem }}m +-Xmx{{ historical_mem }}m +-XX:MaxDirectMemorySize={{ historical_MaxDirectMemorySize }}m +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Dlogfile.name=historical +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager + diff --git a/parcels/roles/components/druid/templates/historical_runtime.j2 b/parcels/roles/components/druid/templates/historical_runtime.j2 new file mode 100644 index 0000000..cd8b21c --- /dev/null +++ b/parcels/roles/components/druid/templates/historical_runtime.j2 @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/historical +druid.plaintextPort=8083 + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers +druid.processing.buffer.sizeBytes={{ historical_buffer_sizeBytes }} +druid.processing.numMergeBuffers={{ historical_numMergeBuffers }} +druid.processing.numThreads={{ historical_numThreads }} +druid.processing.tmpDir=var/druid/processing + +# Segment storage +#druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":3000000000000}] +#druid.server.maxSize=35000000000000 +druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":{{ segmentCache_max_size }}}] +druid.server.maxSize={{ server_disk_maxsize }} + +# Query cache +druid.historical.cache.useCache=true +druid.historical.cache.populateCache=true +druid.cache.type=caffeine +druid.cache.sizeInBytes=256000000 + +druid.query.groupBy.maxMergingDictionarySize=10000000000 +druid.query.groupBy.maxOnDiskStorage=10000000000 diff --git a/parcels/roles/components/druid/templates/keepdruidall.j2 b/parcels/roles/components/druid/templates/keepdruidall.j2 new file mode 100644 index 0000000..071008d --- /dev/null +++ b/parcels/roles/components/druid/templates/keepdruidall.j2 @@ -0,0 +1,42 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 40 60 +# description: keepdruidall +source /etc/profile +PRO_NAME=keepdruidall + +INS_DIR={{ install_path }} +#版本 +VERSION={{ druid_version }} + +case $1 in +start) +master=`ps -ef | grep dae-druid-all.sh | grep -v grep | wc -l` +if [ $master -lt 1 ];then +nohup $INS_DIR/$VERSION/bin/dae-druid-all.sh > /dev/null 2>&1 & +fi +;; + +stop) +keeppid=`ps -ef | grep dae-druid-all.sh | grep -v grep | awk '{print $2}'` +echo "守护进程PID：$keeppid" +kill -9 $keeppid +echo "关闭所有druid进程" +#ps aux | grep druid | grep -v grep | awk '{print $2}' | xargs kill -9 +livenum=`ps -ef |grep "druid" | grep -v grep | grep -v json | wc -l` +if [ $livenum -ne 0 ];then +ps aux|grep "druid" |grep -v grep | awk '{print $2}'| xargs kill -9 +fi +;; + +status) +ps -ef | grep druid | grep -v grep | grep -v json +;; + +* ) +echo "use keepdruidall [start|stop|status]" +;; +esac + diff --git a/parcels/roles/components/druid/templates/keepdruiddata.j2 b/parcels/roles/components/druid/templates/keepdruiddata.j2 new file mode 100644 index 0000000..99cfd40 --- /dev/null +++ b/parcels/roles/components/druid/templates/keepdruiddata.j2 @@ -0,0 +1,42 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 40 60 +# description: keepdruiddata +source /etc/profile +PRO_NAME=keepdruiddata + +INS_DIR={{ install_path }} +#版本 +VERSION={{ druid_version }} + +case $1 in +start) +master=`ps -ef | grep dae-druid-data.sh | grep -v grep | wc -l` +if [ $master -lt 1 ];then +nohup $INS_DIR/$VERSION/bin/dae-druid-data.sh > /dev/null 2>&1 & +fi +;; + +stop) +keeppid=`ps -ef | grep dae-druid-data.sh | grep -v grep | awk '{print $2}'` +echo "守护进程PID：$keeppid" +kill -9 $keeppid +echo "关闭所有druid进程" +#ps aux | grep druid | grep -v grep | awk '{print $2}' | xargs kill -9 +livenum=`ps -ef |grep "druid" | grep -v grep | grep -v json | wc -l` +if [ $livenum -ne 0 ];then +ps aux|grep "druid" |grep -v grep | awk '{print $2}'| xargs kill -9 +fi +;; + +status) +ps -ef | grep druid | grep -v grep | grep -v json +;; + +* ) +echo "use keepdruiddata [start|stop|status]" +;; +esac + diff --git a/parcels/roles/components/druid/templates/keepdruidquery.j2 b/parcels/roles/components/druid/templates/keepdruidquery.j2 new file mode 100644 index 0000000..5d63a00 --- /dev/null +++ b/parcels/roles/components/druid/templates/keepdruidquery.j2 @@ -0,0 +1,42 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 40 60 +# description: keepdruidquery +source /etc/profile +PRO_NAME=keepdruidquery + +INS_DIR={{ install_path }} +#版本 +VERSION={{ druid_version }} + +case $1 in +start) +master=`ps -ef | grep dae-druid-query.sh | grep -v grep | wc -l` +if [ $master -lt 1 ];then +nohup $INS_DIR/$VERSION/bin/dae-druid-query.sh > /dev/null 2>&1 & +fi +;; + +stop) +keeppid=`ps -ef | grep dae-druid-query.sh | grep -v grep | awk '{print $2}'` +echo "守护进程PID：$keeppid" +kill -9 $keeppid +echo "关闭所有druid进程" +#ps aux | grep druid | grep -v grep | awk '{print $2}' | xargs kill -9 +livenum=`ps -ef |grep "druid" | grep -v grep | grep -v json | wc -l` +if [ $livenum -ne 0 ];then +ps aux|grep "druid" |grep -v grep | awk '{print $2}'| xargs kill -9 +fi +;; + +status) +ps -ef | grep druid | grep -v grep | grep -v json +;; + +* ) +echo "use keepdruidquery [start|stop|status]" +;; +esac + diff --git a/parcels/roles/components/druid/templates/log4j2.xml.j2 b/parcels/roles/components/druid/templates/log4j2.xml.j2 new file mode 100644 index 0000000..5847523 --- /dev/null +++ b/parcels/roles/components/druid/templates/log4j2.xml.j2 @@ -0,0 +1,76 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. +--> +<Configuration status="WARN"> +<properties> + <property name="pattern">%d{yyyy-MM-dd HH:mm:ss.SSS} %c{1.} %t [%p] %msg%n</property> + <property name="patternNoTime">%msg%n</property> + <property name="patternMetrics">%d %-8r %m%n</property> + <property name="logPath">{{ install_path }}/{{ druid_version }}/var/sv</property> +</properties> +<appenders> + <RollingFile name="STDOUT" + fileName="${logPath}/${sys:logfile.name}.log" + filePattern="${logPath}/history/${sys:logfile.name}-%d{yyyy-MM-dd}.log.%i.gz"> + + <PatternLayout> + <pattern>${pattern}</pattern> + </PatternLayout> + + <Policies> + <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + </Policies> + + <DefaultRolloverStrategy max="10"> + <Delete basePath="${LOG_PATH}/history" maxDepth="1"> + <IfFileName glob="*.gz"> + <IfLastModified age="7d"> + <!--<IfAny> + <IfAccumulatedFileSize exceeds="200 GB" /> + </IfAny>--> + </IfLastModified> + </IfFileName> + </Delete> + </DefaultRolloverStrategy> + + </RollingFile> + +<!-- + <RollingFile name="METRICS" + fileName="${logPath}/${sys:logfile.name}.metrics" + filePattern="${logPath}/old/${sys:logfile.name}-%d{yyyy-MM-dd}.metrics.%i.gz"> + <PatternLayout> + <pattern>${patternMetrics}</pattern> + </PatternLayout> + <Policies> + <SizeBasedTriggeringPolicy size="100 MB"/> + </Policies> + <DefaultRolloverStrategy max="6"/> + </RollingFile> +--> + +</appenders> + +<loggers> + <Root level="error"> + <appender-ref ref="STDOUT"/> + </Root> + +</loggers> +</Configuration> diff --git a/parcels/roles/components/druid/templates/middleManager_runtime.properties.j2 b/parcels/roles/components/druid/templates/middleManager_runtime.properties.j2 new file mode 100644 index 0000000..b953c5e --- /dev/null +++ b/parcels/roles/components/druid/templates/middleManager_runtime.properties.j2 @@ -0,0 +1,43 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/middleManager +druid.plaintextPort=8091 + +# Number of tasks per middleManager +druid.worker.capacity=200 + +# Task launch parameters +druid.indexer.runner.javaOpts=-server {{ middlemanager_runner_javaOpts }} -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dlog4j.configurationFile=conf/druid/cluster/_common/log4j2-task.xml +druid.indexer.task.baseTaskDir=var/druid/task + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers on Peons +druid.indexer.fork.property.druid.processing.numMergeBuffers={{ middlemanager_numMergeBuffers }} +druid.indexer.fork.property.druid.processing.buffer.sizeBytes={{ middlemanager_buffer_sizeBytes }} +druid.indexer.fork.property.druid.processing.numThreads={{ middlemanager_numThreads }} + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp + +druid.query.groupBy.maxMergingDictionarySize=10000000000 +druid.query.groupBy.maxOnDiskStorage=10000000000 + diff --git a/parcels/roles/components/druid/templates/set_druid_env.sh.j2 b/parcels/roles/components/druid/templates/set_druid_env.sh.j2 new file mode 100644 index 0000000..21a4f0a --- /dev/null +++ b/parcels/roles/components/druid/templates/set_druid_env.sh.j2 @@ -0,0 +1,30 @@ +#!/bin/bash + + +#num=`cat /etc/profile | grep druid | wc -l` + +#if [ $num -eq "0" ];then +# echo -e "\n#druid" >> /etc/profile +# echo -e "export DRUID_HOME={{ install_path }}/{{ druid_version }}" >> /etc/profile +# echo -e "export PATH=\$DRUID_HOME/bin:\$PATH" >> /etc/profile +# source /etc/profile +#fi + +keeppath='/etc/init.d/keepdruidall' + if [ -x $keeppath ];then + chkconfig --add keepdruidall + chkconfig keepdruidall on + fi + +keeppath='/etc/init.d/keepdruiddata' + if [ -x $keeppath ];then + chkconfig --add keepdruiddata + chkconfig keepdruiddata on + fi + +keeppath='/etc/init.d/keepdruidquery' + if [ -x $keeppath ];then + chkconfig --add keepdruidquery + chkconfig keepdruidquery on + fi + diff --git a/parcels/roles/components/druid/templates/unload_druid.sh.j2 b/parcels/roles/components/druid/templates/unload_druid.sh.j2 new file mode 100644 index 0000000..173167e --- /dev/null +++ b/parcels/roles/components/druid/templates/unload_druid.sh.j2 @@ -0,0 +1,43 @@ +#!/bin/bash +source /etc/profile + +function killService(){ +keeppath='/etc/init.d/keepdruidall' +if [ -x $keeppath ];then +service keepdruidall stop +chkconfig keepdruidall off +systemctl daemon-reload +rm -rf /etc/init.d/keepdruidall +fi + +keeppath='/etc/init.d/keepdruiddata' +if [ -x $keeppath ];then +service keepdruiddata stop +chkconfig keepdruiddata off +systemctl daemon-reload +rm -rf /etc/init.d/keepdruiddata +fi + +keeppath='/etc/init.d/keepdruidquery' +if [ -x $keeppath ];then +service keepdruidquery stop +chkconfig keepdruidquery off +systemctl daemon-reload +rm -rf /etc/init.d/keepdruidquery +fi + +} + +function killPid(){ + +livenum=`ps -ef |grep "druid" | grep -v grep | grep -v json | wc -l` +if [ $livenum -ne 0 ];then +ps aux|grep "druid" |grep -v grep | awk '{print $2}'| xargs kill -9 +fi + +} + +killService +sleep 15 +killPid +rm -rf {{ install_path }}/{{ druid_version }} diff --git a/parcels/roles/components/druid/vars/main.yml b/parcels/roles/components/druid/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/components/druid/vars/main.yml diff --git a/parcels/roles/components/hadoop/files/get_qjournal.sh b/parcels/roles/components/hadoop/files/get_qjournal.sh new file mode 100755 index 0000000..5767a5d --- /dev/null +++ b/parcels/roles/components/hadoop/files/get_qjournal.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +hdfsip=`pwd`/roles/commons/hadoop/files/iplist + + +jourzk=`sed -n '1,1p' $hdfsip`\:8485\;`sed -n '2,2p' $hdfsip`\:8485\;`sed -n '3,3p' $hdfsip`\:8485 + + +echo $jourzk diff --git a/parcels/roles/components/hadoop/files/iplist b/parcels/roles/components/hadoop/files/iplist new file mode 100644 index 0000000..9618e84 --- /dev/null +++ b/parcels/roles/components/hadoop/files/iplist @@ -0,0 +1,3 @@ +192.168.44.33 +192.168.44.34 +192.168.44.35 diff --git a/parcels/roles/components/hadoop/tasks/install_hadoop.yml b/parcels/roles/components/hadoop/tasks/install_hadoop.yml new file mode 100644 index 0000000..1f296f3 --- /dev/null +++ b/parcels/roles/components/hadoop/tasks/install_hadoop.yml @@ -0,0 +1,310 @@ +- name: node_nums to ansible variable + set_fact: node_nums="{{groups.hadoop|length}}" + +- name: To terminate execution + fail: + msg: "Hadoop 集群最少需要配置3个节点，请修改configurations/hosts内对应IP列表。" + when: node_nums < '3' + +- name: check jdk 1.8_73 + shell: source /etc/profile && java -version 2>&1 | grep 1.8.0_73 | wc -l + ignore_errors: false + register: jdk_out + +- name: To terminate execution + fail: + msg: "检测到目标集群内存在JDK未安装节点，请检查" + when: jdk_out.stdout != '2' + run_once: true + delegate_to: 127.0.0.1 + +- name: create hadoop package path:{{ install_path }} + file: + state: directory + path: '{{ install_path }}' + +- block: + - name: Check if the Hadoop service already exists + shell: source /etc/profile && jps -l | egrep "org.apache.hadoop.hdfs.qjournal.server.JournalNode|org.apache.hadoop.hdfs.tools.DFSZKFailoverController|org.apache.hadoop.hdfs.server.datanode.DataNode|org.apache.hadoop.hdfs.server.namenode.NameNode" | wc -l + register: check_out + - name: copy unload_hadoop.sh to {{ install_path }}/ + template: + src: 'unload_hadoop.sh.j2' + dest: '{{ install_path }}/unload_hadoop.sh' + force: true + mode: 0755 + when: check_out.stdout >= '1' + - name: unload kafka + shell: cd {{ install_path }} && sh unload_hadoop.sh + when: check_out.stdout >= '1' + - name: Ansible delete {{ install_path }}/unload_hadoop.sh + file: + path: "{{ install_path }}/unload_hadoop.sh" + state: absent + - name: Check if the Hadoop service already exists + shell: source /etc/profile && jps -l | egrep "org.apache.hadoop.hdfs.qjournal.server.JournalNode|org.apache.hadoop.hdfs.tools.DFSZKFailoverController|org.apache.hadoop.hdfs.server.datanode.DataNode|org.apache.hadoop.hdfs.server.namenode.NameNode" | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "卸载失败，组件可能非本安装部署，请手动卸载后继续安装" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "yes" + +- block: + - name: Check if the Hadoop service already exists + shell: source /etc/profile && jps -l | egrep "org.apache.hadoop.hdfs.qjournal.server.JournalNode|org.apache.hadoop.hdfs.tools.DFSZKFailoverController|org.apache.hadoop.hdfs.server.datanode.DataNode|org.apache.hadoop.hdfs.server.namenode.NameNode" | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "检测到目标集群内已有Hadoop请检查IP是否正确或停止/卸载已启动的组件" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "no" + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181" + when: '(groups.zookeeper|length) == 3' + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181,{{groups.zookeeper[3]}}:2181,{{groups.zookeeper[4]}}:2181" + when: '(groups.zookeeper|length) >= 5' + +- name: create hadoop package path:{{ data_path }}/hadoop + file: + state: directory + path: '{{ data_path }}/hadoop' + +#复制tar到目标服务器 +- name: copy hadoop-2.7.1.tar.gz + copy: + src: '{{ package_path }}/hadoop-2.7.1.tar.gz' + dest: '{{ install_path }}/' + force: true + backup: yes + +#解压tar +- name: unpack hadoop-2.7.1.tar.gz + unarchive: + src: 'hadoop-2.7.1.tar.gz' + dest: '{{ install_path }}/' + +#获取slaves列表 +- name: get slaves to:{{ install_path }}/{{ hadoop_version }}/etc/hadoop/slaves + shell: echo "{{ ansible_play_hosts_all }}" | grep -E -o "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+" > {{ install_path }}/{{ hadoop_version }}/etc/hadoop/slaves + +- name: get master_ip + shell: sed -n '1,1p' {{ install_path }}/{{ hadoop_version }}/etc/hadoop/slaves + register: master_out + +- name: master_ip to ansible variable + set_fact: master_ip="{{master_out.stdout}}" + +- name: get slave1_ip + shell: sed -n '2,2p' {{ install_path }}/{{ hadoop_version }}/etc/hadoop/slaves + register: slave1_out + +- name: slave1_ip to ansible variable + set_fact: slave1_ip="{{slave1_out.stdout}}" + +- name: get slave2_ip + shell: sed -n '3,3p' {{ install_path }}/{{ hadoop_version }}/etc/hadoop/slaves + register: slave2_out + +- name: slave2_ip to ansible variable + set_fact: slave2_ip="{{slave2_out.stdout}}" + +- name: qjournal servers to ansible variable + set_fact: qjournal_servers="{{groups.hadoop[0]}}:8485;{{groups.hadoop[1]}}:8485;{{groups.hadoop[2]}}:8485" + +#拷贝hdfs 配置文件 +- name: copy hadoop config files + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + backup: yes + with_items: + - { src: 'core-site.xml.j2', dest: '{{ install_path }}/{{ hadoop_version }}/etc/hadoop/core-site.xml' } + - { src: 'hadoop-env.sh.j2', dest: '{{ install_path }}/{{ hadoop_version }}/etc/hadoop/hadoop-env.sh' } + - { src: 'hdfs-site.xml.j2', dest: '{{ install_path }}/{{ hadoop_version }}/etc/hadoop/hdfs-site.xml' } + - { src: 'mapred-site.xml.bak.j2', dest: '{{ install_path }}/{{ hadoop_version }}/etc/hadoop/mapred-site.xml.bak' } + - { src: 'yarn-site.xml.j2', dest: '{{ install_path }}/{{ hadoop_version }}/etc/hadoop/yarn-site.xml' } + +#拷贝hdfs matser守护及开机自启脚本 +- name: copy hadoop master files + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + mode: 0755 + backup: yes + with_items: + - { src: 'dae-hdfsjournal.sh.j2', dest: '{{ install_path }}/{{ hadoop_version }}/sbin/dae-hdfsjournal.sh' } + - { src: 'dae-hdfsmaster.sh.j2', dest: '{{ install_path }}/{{ hadoop_version }}/sbin/dae-hdfsmaster.sh' } + - { src: 'keephdfsmaster.j2', dest: '/etc/init.d/keephdfsmaster' } + - { src: 'keephdfsjournal.j2', dest: '/etc/init.d/keephdfsjournal' } + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + +#拷贝hdfs slave1_ip守护及开机自启脚本 +- name: copy hadoop slave1_ip files + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + mode: 0755 + backup: yes + with_items: + - { src: 'dae-hdfsjournal.sh.j2', dest: '{{ install_path }}/{{ hadoop_version }}/sbin/dae-hdfsjournal.sh' } + - { src: 'dae-hdfsslave.sh.j2', dest: '{{ install_path }}/{{ hadoop_version }}/sbin/dae-hdfsslave.sh' } + - { src: 'keephdfsslave.j2', dest: '/etc/init.d/keephdfsslave' } + - { src: 'keephdfsjournal.j2', dest: '/etc/init.d/keephdfsjournal' } + run_once: true + delegate_facts: true + delegate_to: "{{ slave1_ip }}" + +#拷贝hdfs slave2_ip守护及开机自启脚本 +- name: copy hadoop slave2_ip files + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + mode: 0755 + backup: yes + with_items: + - { src: 'dae-hdfsjournal.sh.j2', dest: '{{ install_path }}/{{ hadoop_version }}/sbin/dae-hdfsjournal.sh' } + - { src: 'keephdfsjournal.j2', dest: '/etc/init.d/keephdfsjournal' } + run_once: true + delegate_facts: true + delegate_to: "{{ slave2_ip }}" + +#拷贝hdfs 其他工作节点守护及开机自启脚本 +- name: copy hadoop worker files + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + mode: 0755 + backup: yes + with_items: + - { src: 'dae-hdfsworker.sh.j2', dest: '{{ install_path }}/{{ hadoop_version }}/sbin/dae-hdfsworker.sh' } + - { src: 'keephdfsworker.j2', dest: '/etc/init.d/keephdfsworker' } + +#拷贝 设置环境变量脚本 +- name: copy set_hadoop_env.sh path:{{ install_path }}/{{ hadoop_version }}/bin/ + template: + src: 'set_hadoop_env.sh.j2' + dest: '{{ install_path }}/{{ hadoop_version }}/bin/set_hadoop_env.sh' + mode: 0755 + backup: yes + +#配置环境变量 +- name: set hadoop env + shell: cd {{ install_path }}/{{ hadoop_version }}/bin/ && ./set_hadoop_env.sh + +- name: start hadoop journal + shell: source /etc/profile && sh /etc/init.d/keephdfsjournal start + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ master_ip }}' } + - { ip: '{{ slave1_ip }}' } + - { ip: '{{ slave2_ip }}' } + +- name: start {{ master_ip }} journal + shell: source /etc/profile && jps | grep JournalNode | grep -v grep | wc -l + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + register: status_out1 + +- name: start {{ slave1_ip }} journal + shell: source /etc/profile && jps | grep JournalNode | grep -v grep | wc -l + run_once: true + delegate_facts: true + delegate_to: "{{ slave1_ip }}" + register: status_out2 + +- name: start {{ slave2_ip }} journal + shell: source /etc/profile && jps | grep JournalNode | grep -v grep | wc -l + run_once: true + delegate_facts: true + delegate_to: "{{ slave2_ip }}" + register: status_out3 + +- name: copy hadoop initialize files + template: + src: 'ini_hadoop.sh.j2' + dest: '{{ install_path }}/{{ hadoop_version }}/bin/ini_hadoop.sh' + backup: yes + mode: 0755 + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + +- name: Waiting for the JournalNode start,sleep 30s + shell: sleep 30 + +- fail: + msg: "JournalNode节点启动异常，请检查JournalNode" + when: status_out1.stdout != '1' and status_out2.stdout != '1' and status_out3.stdout != '1' + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + +- name: + shell: sh {{ install_path }}/{{ hadoop_version }}/bin/ini_hadoop.sh 1 | grep "yes" | grep -v grep | wc -l + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + register: ini_out + when: status_out1.stdout == '1' and status_out2.stdout == '1' and status_out3.stdout == '1' + +- fail: + msg: "namenode 初始化异常" + when: ini_out.stdout != '1' + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + +- name: + shell: sh {{ install_path }}/{{ hadoop_version }}/bin/ini_hadoop.sh 2 | grep "yes" | grep -v grep | wc -l + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + register: ini_out2 + when: ini_out.stdout == '1' + +- fail: + msg: "hadoop-zk 初始化异常" + when: ini_out2.stdout != '1' + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + +- name: start hadoop-master + shell: service keephdfsmaster start + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + +- name: Waiting for the Master-namenode start,sleep 30s + shell: sleep 30 + +- name: copy namenode files to Standby + shell: "yes | {{ install_path }}/{{ hadoop_version }}/bin/hdfs namenode -bootstrapStandby" + run_once: true + delegate_facts: true + delegate_to: "{{ slave1_ip }}" + +- name: start hadoop-slave + shell: service keephdfsslave start + run_once: true + delegate_facts: true + delegate_to: "{{ slave1_ip }}" + +- name: start hadoop-worker + shell: service keephdfsworker start + +- name: deleted hadoop.tar.gz + shell: rm -rf {{ install_path }}/hadoop-2.7.1.tar.gz diff --git a/parcels/roles/components/hadoop/tasks/main.yml b/parcels/roles/components/hadoop/tasks/main.yml new file mode 100644 index 0000000..89353c0 --- /dev/null +++ b/parcels/roles/components/hadoop/tasks/main.yml @@ -0,0 +1 @@ +- include: install_hadoop.yml diff --git a/parcels/roles/components/hadoop/templates/core-site.xml.j2 b/parcels/roles/components/hadoop/templates/core-site.xml.j2 new file mode 100644 index 0000000..b1e3956 --- /dev/null +++ b/parcels/roles/components/hadoop/templates/core-site.xml.j2 @@ -0,0 +1,59 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> + <property> + <name>fs.defaultFS</name> + <value>hdfs://ns1</value> + </property> + <property> + <name>hadoop.tmp.dir</name> + <value>file:{{ data_path }}/hadoop/tmp</value> + </property> + <property> + <name>io.file.buffer.size</name> + <value>131702</value> + </property> + <property> + <name>hadoop.proxyuser.root.hosts</name> + <value>*</value> + </property> + <property> + <name>hadoop.proxyuser.root.groups</name> + <value>*</value> + </property> + <property> + <name>hadoop.logfile.size</name> + <value>10000000</value> + <description>The max size of each log file</description> + </property> + <property> + <name>hadoop.logfile.count</name> + <value>1</value> + <description>The max number of log files</description> + </property> + <property> + <name>ha.zookeeper.quorum</name> + <value>{{ zookeeper_servers }}</value> + </property> + <property> + <name>ipc.client.connect.timeout</name> + <value>90000</value> + </property> +</configuration> + diff --git a/parcels/roles/components/hadoop/templates/dae-hdfsjournal.sh.j2 b/parcels/roles/components/hadoop/templates/dae-hdfsjournal.sh.j2 new file mode 100644 index 0000000..e25518d --- /dev/null +++ b/parcels/roles/components/hadoop/templates/dae-hdfsjournal.sh.j2 @@ -0,0 +1,21 @@ +#!/bin/bash + +BASH_DIR={{ install_path }} +VERSION={{ hadoop_version }} +source /etc/profile + +while true ; do + + jn=`ps -ef | grep JournalNode | grep -v grep | wc -l` + + if [ $jn -eq "0" ];then + yes | $BASH_DIR/$VERSION/sbin/hadoop-daemon.sh start journalnode > /dev/null + OLD_NUM=`cat $BASH_DIR/$VERSION/protect/restartsum/JournalNode` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASH_DIR/$VERSION/protect/restartsum/JournalNode + echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart JournalNode - 重启次数 -> $RESTART_NUM." >> $BASH_DIR/$VERSION/protect/protecthdfs.log + fi + +sleep 60 +done + diff --git a/parcels/roles/components/hadoop/templates/dae-hdfsmaster.sh.j2 b/parcels/roles/components/hadoop/templates/dae-hdfsmaster.sh.j2 new file mode 100644 index 0000000..586695c --- /dev/null +++ b/parcels/roles/components/hadoop/templates/dae-hdfsmaster.sh.j2 @@ -0,0 +1,38 @@ +#!/bin/bash +source /etc/profile + +BASH_DIR={{ install_path }} +VERSION={{ hadoop_version }} + +while true ; do + nn=`ps -ef | grep NameNode | grep -v grep | wc -l` + dc=`ps -ef | grep DFSZKFailoverController | grep -v grep | wc -l` +# nm=`ps -ef | grep NodeManager | grep -v grep | wc -l` + + if [ $nn -eq "0" ];then + yes | $BASH_DIR/$VERSION/sbin/hadoop-daemon.sh start namenode > /dev/null + OLD_NUM=`cat $BASH_DIR/$VERSION/protect/restartsum/NameNode` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASh_DIR/$VERSION/protect/restartsum/NameNode + echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart NameNode - 重启次数 -> $RESTART_NUM." >> $BASH_DIR/$VERSION/protect/protecthdfs.log + fi + + if [ $dc -eq "0" ];then + yes | $BASH_DIR/$VERSION/sbin/hadoop-daemon.sh start zkfc > /dev/null + OLD_NUM=`cat $BASH_DIR/$VERSION/protect/restartsum/ZKFC` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASH_DIR/$VERSION/protect/restartsum/ZKFC + echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart DFSZKFailoverController - 重启次数 -> $RESTART_NUM." >> $BASH_DIR/$VERSION/protect/protecthdfs.log + fi + +# if [ $nm -eq "0" ];then +# $BASH_DIR/$VERSION/sbin/yarn-daemon.sh start nodemanager > /dev/null +# OLD_NUM=`cat $BASH_DIR/$VERSION/protect/restartsum/NodeManager` +# RESTART_NUM=`expr $OLD_NUM + 1` +# echo $RESTART_NUM > $BASH_DIR/$VERSION/protect/restartsum/NodeManager +# echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart NodeManager - 重启次数 -> $RESTART_NUM." >> $BASH_DIR/$VERSION/protect/protecthdfs.log +# fi + +sleep 60 +done + diff --git a/parcels/roles/components/hadoop/templates/dae-hdfsslave.sh.j2 b/parcels/roles/components/hadoop/templates/dae-hdfsslave.sh.j2 new file mode 100644 index 0000000..0612471 --- /dev/null +++ b/parcels/roles/components/hadoop/templates/dae-hdfsslave.sh.j2 @@ -0,0 +1,48 @@ +#!/bin/bash + +BASH_DIR={{ install_path }} +VERSION={{ hadoop_version }} +source /etc/profile + +while true ; do + nn=`ps -ef | grep NameNode | grep -v grep | wc -l` + dc=`ps -ef | grep DFSZKFailoverController | grep -v grep | wc -l` +# nm=`ps -ef | grep NodeManager | grep -v grep | wc -l` +# rem=`ps -ef | grep ResourceManager | grep -v grep | wc -l` + + if [ $nn -eq "0" ];then + yes | $BASH_DIR/$VERSION/sbin/hadoop-daemon.sh start namenode > /dev/null + OLD_NUM=`cat $BASH_DIR/$VERSION/protect/restartsum/NameNode` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASH_DIR/$VERSION/protect/restartsum/NameNode + echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart NameNode - 重启次数 -> $RESTART_NUM." >> $BASH_DIR/$VERSION/protect/protecthdfs.log + fi + + if [ $dc -eq "0" ];then + yes | $BASH_DIR/$VERSION/sbin/hadoop-daemon.sh start zkfc > /dev/null + OLD_NUM=`cat $BASH_DIR/$VERSION/protect/restartsum/ZKFC` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASH_DIR/$VERSION/protect/restartsum/ZKFC + echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart DFSZKFailoverController - 重>启次数 -> $RESTART_NUM." >> $BASH_DIR/$VERSION/protect/protecthdfs.log + fi + + +# if [ $rem -eq "0" ];then +# $BASH_DIR/$VERSION/sbin/yarn-daemon.sh start resourcemanager > /dev/null +# OLD_NUM=`cat $BASH_DIR/$VERSION/protect/restartsum/ResourceManager` +# RESTART_NUM=`expr $OLD_NUM + 1` +# echo $RESTART_NUM > $BASH_DIR/$VERSION/protect/restartsum/ResourceManager +# echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart ResourceManager - 重启次数 -> $RESTART_NUM." >> $BASH_DIR/$VERSION/protect/protecthdfs.log +# fi +# +# if [ $nm -eq "0" ];then +# $BASH_DIR/$VERSION/sbin/yarn-daemon.sh start nodemanager > /dev/null +# OLD_NUM=`cat $BASH_DIR/$VERSION/protect/restartsum/NodeManager` +# RESTART_NUM=`expr $OLD_NUM + 1` +# echo $RESTART_NUM > $BASH_DIR/$VERSION/protect/restartsum/NodeManager +# echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart NodeManager - 重启次数 -> $RESTART_NUM." >> $BASH_DIR/$VERSION/protect/protecthdfs.log +# fi + +sleep 60 +done + diff --git a/parcels/roles/components/hadoop/templates/dae-hdfsworker.sh.j2 b/parcels/roles/components/hadoop/templates/dae-hdfsworker.sh.j2 new file mode 100644 index 0000000..b0b6125 --- /dev/null +++ b/parcels/roles/components/hadoop/templates/dae-hdfsworker.sh.j2 @@ -0,0 +1,31 @@ +#!/bin/bash +source /etc/profile + +BASH_DIR={{ install_path }} +VERSION={{ hadoop_version }} + + +while true ; do + dn=`ps -ef | grep DataNode | grep -v grep | wc -l` +# nm=`ps -ef | grep NodeManager | grep -v grep | wc -l` + + + if [ $dn -eq "0" ];then + yes | $BASH_DIR/$VERSION/sbin/hadoop-daemons.sh start datanode > /dev/null + OLD_NUM=`cat $BASH_DIR/$VERSION/protect/restartsum/DataNode` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASH_DIR/$VERSION/protect/restartsum/DataNode + echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart DataNode - 重启次数 -> $RESTART_NUM." >> $BASH_DIR/$VERSION/protect/protecthdfs.log + fi + + #if [ $nm -eq "0" ];then + #$BASH_DIR/$VERSION/sbin/yarn-daemon.sh start nodemanager > /dev/null + #OLD_NUM=`cat $BASH_DIR/$VERSION/protect/restartsum/NodeManager` + #RESTART_NUM=`expr $OLD_NUM + 1` + #echo $RESTART_NUM > $BASH_DIR/$VERSION/protect/restartsum/NodeManager + #echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart NodeManager - 重启次数 -> $RESTART_NUM." >> $BASH_DIR/$VERSION/protect/protecthdfs.log + #fi + +sleep 60 +done + diff --git a/parcels/roles/components/hadoop/templates/hadoop-env.sh.j2 b/parcels/roles/components/hadoop/templates/hadoop-env.sh.j2 new file mode 100644 index 0000000..b03d5ff --- /dev/null +++ b/parcels/roles/components/hadoop/templates/hadoop-env.sh.j2 @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set Hadoop-specific environment variables here. + +# The only required environment variable is JAVA_HOME. All others are +# optional. When running a distributed configuration it is best to +# set JAVA_HOME in this file, so that it is correctly defined on +# remote nodes. +export HADOOP_NAMENODE_JMX_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -javaagent:{{ install_path }}/{{ hadoop_version }}/monitor/jmx_prometheus_javaagent-0.12.0.jar=9905:{{ install_path }}/{{ hadoop_version }}/monitor/hdfs.yaml" +export HADOOP_DATANODE_JMX_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -javaagent:{{ install_path }}/{{ hadoop_version }}/monitor/jmx_prometheus_javaagent-0.12.0.jar=9906:{{ install_path }}/{{ hadoop_version }}/monitor/hdfs.yaml" + +# The java implementation to use. +#export HADOOP_HEAPSIZE=m +#export JAVA_HOME=/usr/local/jdk/jdk1.8.0_73 +export JAVA_HOME={{ java_home }} +# The jsvc implementation to use. Jsvc is required to run secure datanodes +# that bind to privileged ports to provide authentication of data transfer +# protocol. Jsvc is not required if SASL is configured for authentication of +# data transfer protocol using non-privileged ports. +#export JSVC_HOME=${JSVC_HOME} + +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"} + +# Extra Java CLASSPATH elements. Automatically insert capacity-scheduler. +for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do + if [ "$HADOOP_CLASSPATH" ]; then + export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f + else + export HADOOP_CLASSPATH=$f + fi +done + +# The maximum amount of heap to use, in MB. Default is 1000. +#export HADOOP_HEAPSIZE= +#export HADOOP_NAMENODE_INIT_HEAPSIZE="" + +# Extra Java runtime options. Empty by default. +export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true" + +# Command specific options appended to HADOOP_OPTS when specified +export HADOOP_NAMENODE_OPTS="-Xmx{{ namenode_mem }}m -Xms{{ namenode_mem }}m -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS" +export HADOOP_DATANODE_OPTS="-Xmx{{ datanode_mem }}m -Xms{{ datanode_mem }}m -Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS" + +export HADOOP_JOURNALNODE_OPTS="$HADOOP_JOURNALNODE_OPTS -Xmx{{ journal_mem }}m -Xms{{ journal_mem }}m" + +export HADOOP_ZKFC_OPTS="$HADOOP_ZKFC_OPTS -Xmx{{ zkfc_mem }}m -Xms{{ zkfc_mem }}m" + +export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS" + +export HADOOP_NFS3_OPTS="$HADOOP_NFS3_OPTS" +export HADOOP_PORTMAP_OPTS="-Xmx512m $HADOOP_PORTMAP_OPTS" + +# The following applies to multiple commands (fs, dfs, fsck, distcp etc) +export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS" +#HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData $HADOOP_JAVA_PLATFORM_OPTS" + +# On secure datanodes, user to run the datanode as after dropping privileges. +# This **MUST** be uncommented to enable secure HDFS if using privileged ports +# to provide authentication of data transfer protocol. This **MUST NOT** be +# defined if SASL is configured for authentication of data transfer protocol +# using non-privileged ports. +export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER} + +# Where log files are stored. $HADOOP_HOME/logs by default. +#export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER + +# Where log files are stored in the secure data environment. +export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER} + +### +# HDFS Mover specific parameters +### +# Specify the JVM options to be used when starting the HDFS Mover. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# export HADOOP_MOVER_OPTS="" + +### +# Advanced Users Only! +### + +# The directory where pid files are stored. /tmp by default. +# NOTE: this should be set to a directory that can only be written to by +# the user that will run the hadoop daemons. Otherwise there is the +# potential for a symlink attack. +export HADOOP_PID_DIR=${HADOOP_PID_DIR} +export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR} + +# A string representing this instance of hadoop. $USER by default. +export HADOOP_IDENT_STRING=$USER diff --git a/parcels/roles/components/hadoop/templates/hdfs-site.xml.j2 b/parcels/roles/components/hadoop/templates/hdfs-site.xml.j2 new file mode 100644 index 0000000..0afda58 --- /dev/null +++ b/parcels/roles/components/hadoop/templates/hdfs-site.xml.j2 @@ -0,0 +1,142 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> + <property> + <name>dfs.namenode.name.dir</name> + <value>file:{{ data_path }}/hadoop/dfs/name</value> + </property> + <property> + <name>dfs.datanode.data.dir</name> + <value>file:{{ data_path }}/hadoop/dfs/data</value> + </property> + <property> + <name>dfs.replication</name> + <value>2</value> + </property> + <property> + <name>dfs.webhdfs.enabled</name> + <value>true</value> + </property> + <property> + <name>dfs.permissions</name> + <value>false</value> + </property> + <property> + <name>dfs.permissions.enabled</name> + <value>false</value> + </property> + <property> + <name>dfs.nameservices</name> + <value>ns1</value> + </property> + <property> + <name>dfs.blocksize</name> + <value>134217728</value> + </property> + <property> + <name>dfs.ha.namenodes.ns1</name> + <value>nn1,nn2</value> + </property> + <!-- nn1的RPC通信地址，nn1所在地址 --> + <property> + <name>dfs.namenode.rpc-address.ns1.nn1</name> + <value>{{ master_ip }}:9000</value> + </property> + <!-- nn1的http通信地址，外部访问地址 --> + <property> + <name>dfs.namenode.http-address.ns1.nn1</name> + <value>{{ master_ip }}:50070</value> + </property> + <!-- nn2的RPC通信地址，nn2所在地址 --> + <property> + <name>dfs.namenode.rpc-address.ns1.nn2</name> + <value>{{ slave1_ip }}:9000</value> + </property> + <!-- nn2的http通信地址，外部访问地址 --> + <property> + <name>dfs.namenode.http-address.ns1.nn2</name> + <value>{{ slave1_ip }}:50070</value> + </property> + <!-- 指定NameNode的元数据在JournalNode日志上的存放位置(一般和zookeeper部署在一起) --> + <property> + <name>dfs.namenode.shared.edits.dir</name> + <value>qjournal://{{ qjournal_servers }}/ns1</value> + </property> + <!-- 指定JournalNode在本地磁盘存放数据的位置 --> + <property> + <name>dfs.journalnode.edits.dir</name> + <value>{{ data_path }}/hadoop/journal</value> + </property> + <!--客户端通过代理访问namenode，访问文件系统，HDFS 客户端与Active 节点通信的Java 类，使用其确定Active 节点是否活跃 --> + <property> + <name>dfs.client.failover.proxy.provider.ns1</name> + <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value> + </property> + <!--这是配置自动切换的方法，有多种使用方法，具体可以看官网，在文末会给地址，这里是远程登录杀死的方法 --> + <property> + <name>dfs.ha.fencing.methods</name> + <value>sshfence</value> + <value>shell(true)</value> + </property> + <!-- 这个是使用sshfence隔离机制时才需要配置ssh免登陆 --> + <property> + <name>dfs.ha.fencing.ssh.private-key-files</name> + <value>/root/.ssh/id_rsa</value> + </property> + <!-- 配置sshfence隔离机制超时时间，这个属性同上，如果你是用脚本的方法切换，这个应该是可以不配置的 --> + <property> + <name>dfs.ha.fencing.ssh.connect-timeout</name> + <value>30000</value> + </property> + <!-- 这个是开启自动故障转移，如果你没有自动故障转移，这个可以先不配 --> + <property> + <name>dfs.ha.automatic-failover.enabled</name> + <value>true</value> + </property> + <property> + <name>dfs.datanode.max.transfer.threads</name> + <value>8192</value> + </property> + <!-- namenode处理RPC请求线程数，增大该值资源占用不大 --> + <property> + <name>dfs.namenode.handler.count</name> + <value>{{ namenode_handlers }}</value> + </property> + <!-- datanode处理RPC请求线程数，增大该值会占用更多内存 --> + <property> + <name>dfs.datanode.handler.count</name> + <value>{{ datanode_handlers }}</value> + </property> + <!-- balance时可占用的带宽 --> + <property> + <name>dfs.balance.bandwidthPerSec</name> + <value>104857600</value> + </property> + <!-- 磁盘预留空间，该空间不会被hdfs占用，单位字节--> + <property> + <name>dfs.datanode.du.reserved</name> + <value>53687091200</value> + </property> + <!-- datanode与namenode连接超时时间，单位毫秒 2 * heartbeat.recheck.interval + 30000 --> + <property> + <name>heartbeat.recheck.interval</name> + <value>100000</value> + </property> +</configuration> + diff --git a/parcels/roles/components/hadoop/templates/ini_hadoop.sh.j2 b/parcels/roles/components/hadoop/templates/ini_hadoop.sh.j2 new file mode 100644 index 0000000..5e19900 --- /dev/null +++ b/parcels/roles/components/hadoop/templates/ini_hadoop.sh.j2 @@ -0,0 +1,46 @@ +#!/bin/bash + + +MASTER_IP={{ master_ip }} +SLAVE1_IP={{ slave1_ip }} + +BASE_DIR={{ install_path }} +VERSION={{ hadoop_version }} + +function ini_namenode() { + +cd $BASE_DIR/$VERSION/bin +yes | ./hadoop namenode -format + +if [ $? -eq "0" ];then +# scp -r $BASE_DIR/hadoop/ root@$SLAVE1_IP:$BASE_DIR/ + echo yes +else + echo no +fi +} + +function ini_zk() { + +cd $BASE_DIR/$VERSION/bin +yes | ./hdfs zkfc -formatZK + +if [ $? -eq "0" ];then + echo yes +else + echo no +fi +} + +case $1 in +[1]*) +ini_namenode +break;; +[2]*) +ini_zk +break;; +* ) +echo "请输入已有的指令." +;; +esac + diff --git a/parcels/roles/components/hadoop/templates/keephdfsjournal.j2 b/parcels/roles/components/hadoop/templates/keephdfsjournal.j2 new file mode 100644 index 0000000..e91ddf4 --- /dev/null +++ b/parcels/roles/components/hadoop/templates/keephdfsjournal.j2 @@ -0,0 +1,45 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 40 60 +# description: keephdfsjournal +source /etc/profile +PRO_NAME=keephdfsjournal + +INS_DIR={{ install_path }} +#版本 +VERSION={{ hadoop_version }} + +case $1 in +start) +journal=`ps -ef | grep dae-hdfsjournal.sh | grep -v grep | wc -l` +if [ $journal -lt 1 ];then +nohup $INS_DIR/$VERSION/sbin/dae-hdfsjournal.sh > /dev/null 2>&1 & +fi + +;; + +stop) +keeppid=`ps -ef | grep dae-hdfsjournal.sh | grep -v grep | awk '{print $2}'` +echo "守护进程PID：$keeppid" +kill -9 $keeppid + +sh $INS_DIR/$VERSION/sbin/hadoop-daemon.sh stop journalnode > /dev/null +;; + +status) +num=`ps -ef | grep JournalNode | grep -v grep | wc -l` +if [ "$num" -eq "1" ];then +echo "JournalNode进程已启动" +else +echo "JournalNode进程未启动" +fi + +;; + +* ) +echo "use keephdfsjournal [start|stop|status]" +;; +esac + diff --git a/parcels/roles/components/hadoop/templates/keephdfsmaster.j2 b/parcels/roles/components/hadoop/templates/keephdfsmaster.j2 new file mode 100644 index 0000000..e34e2aa --- /dev/null +++ b/parcels/roles/components/hadoop/templates/keephdfsmaster.j2 @@ -0,0 +1,40 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 40 60 +# description: keephdfsmaster +source /etc/profile +PRO_NAME=keephdfsmaster + +INS_DIR={{ install_path }} +#版本 +VERSION={{ hadoop_version }} + +case $1 in +start) +master=`ps -ef | grep dae-hdfsmaster.sh | grep -v grep | wc -l` +if [ $master -lt 1 ];then +nohup $INS_DIR/$VERSION/sbin/dae-hdfsmaster.sh > /dev/null 2>&1 & +fi +;; + +stop) +keeppid=`ps -ef | grep dae-hdfsmaster.sh | grep -v grep | awk '{print $2}'` +echo "master守护进程PID：$keeppid" +kill -9 $keeppid + +sh $INS_DIR/$VERSION/sbin/hadoop-daemon.sh stop namenode > /dev/null +sh $INS_DIR/$VERSION/sbin/hadoop-daemon.sh stop zkfc > /dev/null +;; + +status) +hdfs haadmin -getServiceState nn1 +hdfs dfsadmin -report +;; + +* ) +echo "use keephdfsmaster [start|stop|status]" +;; +esac + diff --git a/parcels/roles/components/hadoop/templates/keephdfsslave.j2 b/parcels/roles/components/hadoop/templates/keephdfsslave.j2 new file mode 100644 index 0000000..9557fdc --- /dev/null +++ b/parcels/roles/components/hadoop/templates/keephdfsslave.j2 @@ -0,0 +1,40 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 40 60 +# description: keephdfsslave +source /etc/profile +PRO_NAME=keephdfsslave + +INS_DIR={{ install_path }} +#版本 +VERSION={{ hadoop_version }} + +case $1 in +start) +slave=`ps -ef | grep dae-hdfsslave.sh | grep -v grep | wc -l` +if [ $slave -lt 1 ];then +nohup $INS_DIR/$VERSION/sbin/dae-hdfsslave.sh > /dev/null 2>&1 & +fi +;; + +stop) +keeppid=`ps -ef | grep dae-hdfsslave.sh | grep -v grep | awk '{print $2}'` +echo "守护进程PID：$keeppid" +kill -9 $keeppid + +sh $INS_DIR/$VERSION/sbin/hadoop-daemon.sh stop namenode > /dev/null +sh $INS_DIR/$VERSION/sbin/hadoop-daemon.sh stop zkfc > /dev/null +;; + +status) +hdfs haadmin -getServiceState nn2 +hdfs dfsadmin -report +;; + +* ) +echo "use keephdfsslave [start|stop|status]" +;; +esac + diff --git a/parcels/roles/components/hadoop/templates/keephdfsworker.j2 b/parcels/roles/components/hadoop/templates/keephdfsworker.j2 new file mode 100644 index 0000000..a26d3a4 --- /dev/null +++ b/parcels/roles/components/hadoop/templates/keephdfsworker.j2 @@ -0,0 +1,45 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 40 60 +# description: keephdfsworker +source /etc/profile +PRO_NAME=keephdfsworker + +INS_DIR={{ install_path }} +#版本 +VERSION={{ hadoop_version }} + +case $1 in +start) +worker=`ps -ef | grep dae-hdfsworker.sh | grep -v grep | wc -l` +if [ $worker -lt 1 ];then +nohup $INS_DIR/$VERSION/sbin/dae-hdfsworker.sh > /dev/null 2>&1 & +fi + +;; + +stop) +keeppid=`ps -ef | grep dae-hdfsworker.sh | grep -v grep | awk '{print $2}'` +echo "守护进程PID：$keeppid" +kill -9 $keeppid + +sh $INS_DIR/$VERSION/sbin/hadoop-daemon.sh stop datanode > /dev/null +;; + +status) +num=`ps -ef | grep DataNode | grep -v grep | wc -l` +if [ "$num" -eq "1" ];then +echo "DataNode进程已启动" +else +echo "DataNode进程未启动" +fi + +;; + +* ) +echo "use keephdfsworker [start|stop|status]" +;; +esac + diff --git a/parcels/roles/components/hadoop/templates/mapred-site.xml.bak.j2 b/parcels/roles/components/hadoop/templates/mapred-site.xml.bak.j2 new file mode 100644 index 0000000..a719589 --- /dev/null +++ b/parcels/roles/components/hadoop/templates/mapred-site.xml.bak.j2 @@ -0,0 +1,33 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> + <property> + <name>mapreduce.framework.name</name> + <value>yarn</value> + </property> + <property> + <name>mapreduce.jobhistory.address</name> + <value>{{ master_ip }}:10020</value> + </property> + <property> + <name>mapreduce.jobhistory.webapp.address</name> + <value>{{ master_ip }}:19888</value> + </property> +</configuration> + diff --git a/parcels/roles/components/hadoop/templates/set_hadoop_env.sh.j2 b/parcels/roles/components/hadoop/templates/set_hadoop_env.sh.j2 new file mode 100644 index 0000000..b37c117 --- /dev/null +++ b/parcels/roles/components/hadoop/templates/set_hadoop_env.sh.j2 @@ -0,0 +1,37 @@ +#!/bin/bash + + +num=`cat /etc/profile | grep hadoop | wc -l` + +if [ $num -eq "0" ];then + echo -e "\n#hadoop" >> /etc/profile + echo -e "export HADOOP_HOME={{ install_path }}/{{ hadoop_version }}" >> /etc/profile + echo -e "export PATH=\$HADOOP_HOME/sbin:\$PATH" >> /etc/profile + echo -e "export PATH=\$HADOOP_HOME/bin:\$PATH" >> /etc/profile + source /etc/profile +fi + +keeppath='/etc/init.d/keephdfsmaster' + if [ -x $keeppath ];then + chkconfig --add keephdfsmaster + chkconfig keephdfsmaster on + fi + +keeppath='/etc/init.d/keephdfsslave' + if [ -x $keeppath ];then + chkconfig --add keephdfsslave + chkconfig keephdfsslave on + fi + +keeppath='/etc/init.d/keephdfsworker' + if [ -x $keeppath ];then + chkconfig --add keephdfsworker + chkconfig keephdfsworker on + fi + +keeppath='/etc/init.d/keephdfsjournal' + if [ -x $keeppath ];then + chkconfig --add keephdfsjournal + chkconfig keephdfsjournal on + fi + diff --git a/parcels/roles/components/hadoop/templates/unload_hadoop.sh.j2 b/parcels/roles/components/hadoop/templates/unload_hadoop.sh.j2 new file mode 100644 index 0000000..37c4c0c --- /dev/null +++ b/parcels/roles/components/hadoop/templates/unload_hadoop.sh.j2 @@ -0,0 +1,70 @@ +#!/bin/bash +source /etc/profile + +function killService(){ +keeppath='/etc/init.d/keephdfsjournal' +if [ -x $keeppath ];then +service keephdfsjournal stop +chkconfig keephdfsjournal off +systemctl daemon-reload +rm -rf /etc/init.d/keephdfsjournal +fi + +keeppath='/etc/init.d/keephdfsmaster' +if [ -x $keeppath ];then +service keephdfsmaster stop +chkconfig keephdfsmaster off +systemctl daemon-reload +rm -rf /etc/init.d/keephdfsmaster +fi + +keeppath='/etc/init.d/keephdfsslave' +if [ -x $keeppath ];then +service keephdfsslave stop +chkconfig keephdfsslave off +systemctl daemon-reload +rm -rf /etc/init.d/keephdfsslave +fi + +keeppath='/etc/init.d/keephdfsworker' +if [ -x $keeppath ];then +service keephdfsworker stop +chkconfig keephdfsworker off +systemctl daemon-reload +rm -rf /etc/init.d/keephdfsworker +fi +} + + +function killPid(){ +livenum=`jps -l | egrep -w "org.apache.hadoop.hdfs.qjournal.server.JournalNode" | grep -v grep |wc -l` +if [ $livenum -ne 0 ];then +keeppid=`jps -l |egrep -w "org.apache.hadoop.hdfs.qjournal.server.JournalNode" | awk '{print $1}'` +kill -9 $keeppid +fi + +livenum=`jps -l | egrep -w "org.apache.hadoop.hdfs.tools.DFSZKFailoverController" | grep -v grep |wc -l` +if [ $livenum -ne 0 ];then +keeppid=`jps -l |egrep -w "org.apache.hadoop.hdfs.tools.DFSZKFailoverController" | awk '{print $1}'` +kill -9 $keeppid +fi + +livenum=`jps -l | egrep -w "org.apache.hadoop.hdfs.server.datanode.DataNode" | grep -v grep |wc -l` +if [ $livenum -ne 0 ];then +keeppid=`jps -l |egrep -w "org.apache.hadoop.hdfs.server.datanode.DataNode" | awk '{print $1}'` +kill -9 $keeppid +fi + +livenum=`jps -l | egrep -w "org.apache.hadoop.hdfs.server.namenode.NameNode" | grep -v grep |wc -l` +if [ $livenum -ne 0 ];then +keeppid=`jps -l |egrep -w "org.apache.hadoop.hdfs.server.namenode.NameNode" | awk '{print $1}'` +kill -9 $keeppid +fi +} + +killService +sleep 15 +killPid +rm -rf {{ install_path }}/{{ hadoop_version }} +rm -rf {{ install_path }}/hadoop + diff --git a/parcels/roles/components/hadoop/templates/yarn-site.xml.j2 b/parcels/roles/components/hadoop/templates/yarn-site.xml.j2 new file mode 100644 index 0000000..02ee3c0 --- /dev/null +++ b/parcels/roles/components/hadoop/templates/yarn-site.xml.j2 @@ -0,0 +1,56 @@ +<?xml version="1.0"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<configuration> + + <property> + <name>yarn.nodemanager.aux-services</name> + <value>mapreduce_shuffle</value> + </property> + <property> + <name>yarn.resourcemanager.ha.enabled</name> + <value>true</value> + </property> + <!--声明两台resourcemanager的地址--> + <property> + <name>yarn.resourcemanager.cluster-id</name> + <value>rmcluster</value> + </property> + <property> + <name>yarn.resourcemanager.ha.rm-ids</name> + <value>rm1,rm2</value> + </property> + <property> + <name>yarn.resourcemanager.hostname.rm1</name> + <value>{{ slave1_ip }}</value> + </property> + <property> + <name>yarn.resourcemanager.hostname.rm2</name> + <value>{{ slave2_ip }}</value> + </property> + <!--指定zookeeper集群的地址--> + <property> + <name>yarn.resourcemanager.zk-address</name> + <value>{zookeeperurl}</value> + </property> + <!--启用自动恢复，当任务进行一半，rm坏掉，就要启动自动恢复，默认是false--> + <property> + <name>yarn.resourcemanager.recovery.enabled</name> + <value>true</value> + </property> + <property> + <name>yarn.resourcemanager.store.class</name> + <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value> + </property> +</configuration> diff --git a/parcels/roles/components/hadoop/vars/main.yml b/parcels/roles/components/hadoop/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/components/hadoop/vars/main.yml diff --git a/parcels/roles/components/hbase/tasks/install_hbase.yml b/parcels/roles/components/hbase/tasks/install_hbase.yml new file mode 100644 index 0000000..287b585 --- /dev/null +++ b/parcels/roles/components/hbase/tasks/install_hbase.yml @@ -0,0 +1,228 @@ +- name: node_nums to ansible variable + set_fact: node_nums="{{groups.hbase|length}}" + +- name: To terminate execution + fail: + msg: "hbase 集群最少需要配置3个节点，请修改configurations/hosts内对应IP列表。" + when: node_nums < '3' + +- name: check jdk 1.8_73 + shell: source /etc/profile && java -version 2>&1 | grep 1.8.0_73 | wc -l + ignore_errors: false + register: jdk_out + +- name: To terminate execution + fail: + msg: "检测到目标集群内存在JDK未安装节点，请检查" + when: jdk_out.stdout != '2' + run_once: true + delegate_to: 127.0.0.1 + +- name: create hbase package path:{{ install_path }} + file: + state: directory + path: '{{ install_path }}' + +- block: + - name: Check if the HBase service already exists + shell: source /etc/profile && jps -l | egrep "org.apache.hadoop.hbase.regionserver.HRegionServer|org.apache.hadoop.hbase.master.HMaster" | wc -l + register: check_out + - name: copy unload_hbase.sh to {{ install_path }}/ + template: + src: 'unload_hbase.sh.j2' + dest: '{{ install_path }}/unload_hbase.sh' + force: true + mode: 0755 + when: check_out.stdout >= '1' + - name: unload hbase + shell: cd {{ install_path }} && sh unload_hbase.sh + when: check_out.stdout >= '1' + - name: Ansible delete {{ install_path }}/unload_hbase.sh + file: + path: "{{ install_path }}/unload_hbase.sh" + state: absent + - name: Check if the HBase service already exists + shell: source /etc/profile && jps -l | egrep "org.apache.hadoop.hbase.regionserver.HRegionServer|org.apache.hadoop.hbase.master.HMaster" | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "卸载失败，组件可能非本安装部署，请联系开发确认或手动卸载后继续安装" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "yes" + +- block: + - name: Check if the HBase service already exists + shell: source /etc/profile && jps -l | egrep "org.apache.hadoop.hbase.regionserver.HRegionServer|org.apache.hadoop.hbase.master.HMaster" | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "检测到目标集群内已有HBase请检查IP是否正确或停止/卸载已启动的组件" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "no" + +- name: hadoop_ip to ansible variable + set_fact: hadoop_ip="{{groups.hadoop[0]}}" + +- name: check hadoop datanode num ,If the service cannot ,Time out after 60 seconds + shell: source /etc/profile && hadoop dfsadmin -report | grep "Live datanodes" | grep -E -o "[0-9]" + async: 60 + register: datanode_out + run_once: true + delegate_facts: true + delegate_to: "{{ hadoop_ip }}" + +- name: check hadoop namenode num ,If the service cannot ,Time out after 60 seconds + shell: source /etc/profile && hadoop dfsadmin -report |grep 50010 | wc -l + async: 60 + register: namenode_out + run_once: true + delegate_facts: true + delegate_to: "{{ hadoop_ip }}" + +- fail: + msg: "Hadoop 服务节点启动检测异常，请检查服务或是否已做免密登录" + when: datanode_out.stdout <= '1' and namenode_out.stdout <= '1' + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181" + when: '(groups.zookeeper|length) == 3' + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181,{{groups.zookeeper[3]}}:2181,{{groups.zookeeper[4]}}:2181" + when: '(groups.zookeeper|length) >= 5' + +#复制tar到目标服务器 +- name: copy {{ hbase_version }}.tar.gz + copy: + src: '{{ package_path }}/{{ hbase_version }}.tar.gz' + dest: '{{ install_path }}/' + force: true + backup: yes + +#解压tar +- name: unpack {{ hbase_version }}.tar.gz + unarchive: + src: '{{ hbase_version }}.tar.gz' + dest: '{{ install_path }}/' + +#获取 regionservers 列表 +- name: get regionservers to:{{ install_path }}/{{ hbase_version }}/conf/regionservers + shell: echo "{{ ansible_play_hosts_all }}" | grep -E -o "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+" > {{ install_path }}/{{ hbase_version }}/conf/regionservers + +- name: get master_ip + shell: sed -n '1,1p' {{ install_path }}/{{ hbase_version }}/conf/regionservers + register: master_out + +- name: master_ip to ansible variable + set_fact: master_ip="{{master_out.stdout}}" + +- name: get slave1_ip + shell: sed -n '2,2p' {{ install_path }}/{{ hbase_version }}/conf/regionservers + register: slave1_out + +- name: slave1_ip to ansible variable + set_fact: slave1_ip="{{slave1_out.stdout}}" + +- name: get slave2_ip + shell: sed -n '3,3p' {{ install_path }}/{{ hbase_version }}/conf/regionservers + register: slave2_out + +- name: slave2_ip to ansible variable + set_fact: slave2_ip="{{slave2_out.stdout}}" + +- name: get regionservers to:{{ install_path }}/{{ hbase_version }}/conf/backup-masters + shell: "echo {{ slave1_ip }} > {{ install_path }}/{{ hbase_version }}/conf/backup-masters && echo {{ slave2_ip }} >> {{ install_path }}/{{ hbase_version }}/conf/backup-masters" + +#拷贝hbase 配置文件 +- name: copy hbase config files + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + backup: yes + with_items: + - { src: 'hbase-env.sh.j2', dest: '{{ install_path }}/{{ hbase_version }}/conf/hbase-env.sh' } + - { src: 'hbase-site.xml.j2', dest: '{{ install_path }}/{{ hbase_version }}/conf/hbase-site.xml' } + +#拷贝hbase hmaster守护脚本 +- name: copy hbase Hmaster shell + template: + src: 'dae-hmaster.sh.j2' + dest: '{{ install_path }}/{{ hbase_version }}/bin/dae-hmaster.sh' + backup: yes + mode: 0755 + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ master_ip }}' } + - { ip: '{{ slave1_ip }}' } + - { ip: '{{ slave2_ip }}' } + +#拷贝hbase hmaster开机自启 +- name: copy hbase Hmaster keep + template: + src: 'keephbasemaster.j2' + dest: '/etc/init.d/keephbasemaster' + backup: yes + mode: 0755 + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ master_ip }}' } + - { ip: '{{ slave1_ip }}' } + - { ip: '{{ slave2_ip }}' } + +#拷贝hbase region 守护进程和开机自启 +- name: copy hbase region files + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + mode: 0755 + backup: yes + with_items: + - { src: 'dae-hregion.sh.j2', dest: '{{ install_path }}/{{ hbase_version }}/bin/dae-hregion.sh' } + - { src: 'keephbaseregion.j2', dest: '/etc/init.d/keephbaseregion' } + +#拷贝hadoop配置文件到 hbase/conf +- name: copy hadoop-conf to {{ install_path }}/{{ hbase_version }}/conf/ + shell: "scp root@{{ hadoop_ip }}:{{ install_path }}/hadoop-2.7.1/etc/hadoop/{{ item.filename }} {{ install_path }}/{{ hbase_version }}/conf/" + with_items: + - { filename: 'hdfs-site.xml' } + - { filename: 'core-site.xml' } + - { filename: 'yarn-site.xml' } + - { filename: 'mapred-site.xml.bak' } + +#拷贝 设置环境变量脚本 +- name: copy set_hbase_env.sh path:{{ install_path }}/{{ hbase_version }}/bin/ + template: + src: 'set_hbase_env.sh.j2' + dest: '{{ install_path }}/{{ hbase_version }}/bin/set_hbase_env.sh' + mode: 0755 + backup: yes + +#配置环境变量 +- name: set hbase env + shell: cd {{ install_path }}/{{ hbase_version }}/bin/ && sh set_hbase_env.sh + +- name: start hbase master + shell: source /etc/profile && sh /etc/init.d/keephbasemaster 'start' + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ master_ip }}' } + - { ip: '{{ slave1_ip }}' } + - { ip: '{{ slave2_ip }}' } + +- name: start hbase region + shell: source /etc/profile && sh /etc/init.d/keephbaseregion 'start' + +- name: Ansible delete {{ install_path }}/{{ hbase_version }}.tar.gz + file: + path: "{{ install_path }}/{{ hbase_version }}.tar.gz" + state: absent diff --git a/parcels/roles/components/hbase/tasks/main.yml b/parcels/roles/components/hbase/tasks/main.yml new file mode 100644 index 0000000..7ef5aea --- /dev/null +++ b/parcels/roles/components/hbase/tasks/main.yml @@ -0,0 +1 @@ +- include: install_hbase.yml diff --git a/parcels/roles/components/hbase/templates/dae-hmaster.sh.j2 b/parcels/roles/components/hbase/templates/dae-hmaster.sh.j2 new file mode 100644 index 0000000..96a9ec8 --- /dev/null +++ b/parcels/roles/components/hbase/templates/dae-hmaster.sh.j2 @@ -0,0 +1,21 @@ +#!/bin/bash +source /etc/profile + +PRO_NAME=HMaster +BASE_DIR={{ data_path }} +VERSION={{ hbase_version }} + +while true ; do + + NUM=`jps | grep -w $PRO_NAME | grep -v grep |wc -l` + if [ "$NUM" -lt "1" ];then + $BASE_DIR/$VERSION/bin/hbase-daemon.sh start master + OLD_NUM=`cat $BASE_DIR/$VERSION/protect/restartsum/HMaster` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASE_DIR/$VERSION/protect/restartsum/HMaster + echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart HMaster - 重启次数 -> $RESTART_NUM." >> $BASE_DIR/$VERSION/protect/protecthbase.log + + fi + sleep 60 +done + diff --git a/parcels/roles/components/hbase/templates/dae-hregion.sh.j2 b/parcels/roles/components/hbase/templates/dae-hregion.sh.j2 new file mode 100644 index 0000000..4237b61 --- /dev/null +++ b/parcels/roles/components/hbase/templates/dae-hregion.sh.j2 @@ -0,0 +1,20 @@ +#!/bin/bash + +source /etc/profile + +BASE_DIR={{ data_path }} +VERSION={{ hbase_version }} + +while true ; do + + hrnum=`ps -ef | grep HRegionServer | grep -v grep | wc -l` + if [ $hrnum -eq "0" ];then + $BASE_DIR/$VERSION/bin/hbase-daemon.sh start regionserver + HR_OLD_NUM=`cat $BASE_DIR/$VERSION/protect/restartsum/HRegionServer` + HR_RESTART_NUM=`expr $HR_OLD_NUM + 1` + echo $HR_RESTART_NUM > $BASE_DIR/$VERSION/protect/restartsum/HRegionServer + echo "`date +%Y-%m-%d` `date +%H:%M:%S` - restart HRegionServer - 重启次数 -> $HR_RESTART_NUM." >> $BASE_DIR/$VERSION/protect/protecthbase.log + fi + +sleep 60 +done diff --git a/parcels/roles/components/hbase/templates/hbase-env.sh.j2 b/parcels/roles/components/hbase/templates/hbase-env.sh.j2 new file mode 100644 index 0000000..eab1cb6 --- /dev/null +++ b/parcels/roles/components/hbase/templates/hbase-env.sh.j2 @@ -0,0 +1,143 @@ +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Set environment variables here. + +# This script sets variables multiple times over the course of starting an hbase process, +# so try to keep things idempotent unless you want to take an even deeper look +# into the startup scripts (bin/hbase, etc.) + +# The java implementation to use. Java 1.7+ required. +export JAVA_HOME={{ java_home }} + +# Extra Java CLASSPATH elements. Optional. +# export HBASE_CLASSPATH= + +# The maximum amount of heap to use. Default is left to JVM default. +#export HBASE_HEAPSIZE={heap} + +# Uncomment below if you intend to use off heap cache. For example, to allocate 8G of +# offheap, set the value to "8G". +#export HBASE_OFFHEAPSIZE=5G + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export HBASE_OPTS="-XX:+UseConcMarkSweepGC " + +# Configure PermSize. Only needed in JDK7. You can safely remove it for JDK8+ +export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmx{{ region_mem }}m -Xms{{ region_mem }}m -Xmn{{ region_young_mem }}m -Xss256k -XX:MetaspaceSize=512m -XX:MaxMetaspaceSize=512m -XX:SurvivorRatio=2 -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSParallelRemarkEnabled -XX:MaxTenuringThreshold=15 -XX:+UseCMSCompactAtFullCollection -XX:CMSFullGCsBeforeCompaction=1 -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:-DisableExplicitGC" + +export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" + +export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Xmx{{ master_mem }}m -Xms{{ master_mem }}m -javaagent:{{ data_path }}/{{ hbase_version }}/monitor/jmx_prometheus_javaagent-0.12.0.jar=9907:{{ data_path }}/{{ hbase_version }}/monitor/hbase.yaml" + +export HBASE_REGIONSERVER_JMX_OPTS="$HBASE_JMX_BASE -javaagent:{{ data_path }}/{{ hbase_version }}/monitor/jmx_prometheus_javaagent-0.12.0.jar=9908:{{ data_path }}/{{ hbase_version }}/monitor/hbase.yaml" + +# Uncomment one of the below three options to enable java garbage collection logging for the server-side processes. + +# This enables basic gc logging to the .out file. +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + +# This enables basic gc logging to its own file. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:<FILE-PATH>" + +# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:<FILE-PATH> -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" + +# Uncomment one of the below three options to enable java garbage collection logging for the client processes. + +# This enables basic gc logging to the .out file. +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + +# This enables basic gc logging to its own file. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:<FILE-PATH>" + +# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:<FILE-PATH> -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" + +# See the package documentation for org.apache.hadoop.hbase.io.hfile for other configurations +# needed setting up off-heap block caching. + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# NOTE: HBase provides an alternative JMX implementation to fix the random ports issue, please see JMX +# section in HBase Reference Guide for instructions. + +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" +# export HBASE_REST_OPTS="$HBASE_REST_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10105" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers + +# Uncomment and adjust to keep all the Region Server pages mapped to be memory resident +#HBASE_REGIONSERVER_MLOCK=true +#HBASE_REGIONSERVER_UID="hbase" + +# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default. +# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +export HBASE_LOG_DIR={{ data_path }}/{{ hbase_version }}/logs + +# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073" + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +export HBASE_PID_DIR={{ data_path }}/{{ hbase_version }}/pids + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +export HBASE_MANAGES_ZK=false + +# The default log rolling policy is RFA, where the log file is rolled as per the size defined for the +# RFA appender. Please refer to the log4j.properties file to see more details on this appender. +# In case one needs to do log rolling on a date change, one should set the environment property +# HBASE_ROOT_LOGGER to "<DESIRED_LOG LEVEL>,DRFA". +# For example: +#HBASE_ROOT_LOGGER=INFO,DRFA +HBASE_ROOT_LOGGER=ERROR,DRFA +# The reason for changing default to RFA is to avoid the boundary case of filling out disk space as +# DRFA doesn't put any cap on the log size. Please refer to HBase-5655 for more context. diff --git a/parcels/roles/components/hbase/templates/hbase-site.xml.j2 b/parcels/roles/components/hbase/templates/hbase-site.xml.j2 new file mode 100644 index 0000000..bbefdcc --- /dev/null +++ b/parcels/roles/components/hbase/templates/hbase-site.xml.j2 @@ -0,0 +1,174 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>hbase.rootdir</name> + <value>hdfs://ns1/hbase</value> + </property> + + <property> + <name>hbase.cluster.distributed</name> + <value>true</value> + </property> + + <property> + <name>hbase.zookeeper.quorum</name> + <value>{{ zookeeper_servers }}</value> + </property> + +<!-- + <property> + <name>hbase.zookeeper.property.clientPort</name> + <value>2181</value> + </property> +--> + + <property> + <name>hbase.master.info.port</name> + <value>60010</value> + </property> + + <property> + <name>hbase.client.keyvalue.maxsize</name> + <value>1073741824</value> + </property> + + <property> + <name>hbase.server.keyvalue.maxsize</name> + <value>1073741824</value> + </property> + + <property> + <name>zookeeper.znode.parent</name> + <value>/hbase</value> + </property> + + <property> + <name>hbase.rpc.timeout</name> + <value>300000</value> + </property> + + <property> + <name>zookeeper.session.timeout</name> + <value>300000</value> + </property> + + <!--小于该值的文件将在mob compaction中合并--> + <property> + <name>hbase.mob.compaction.mergeable.threshold</name> + <value>1342177280</value> + </property> + + <property> + <name>hbase.mob.file.cache.size</name> + <value>1000</value> + </property> + + <!--mob cache回收缓存周期--> + <property> + <name>hbase.mob.cache.evict.period</name> + <value>3600</value> + </property> + + <!--mob cache回收之后cache中保留文件个数比例，cache数量超过hbase.mob.file.cache.size会回收--> + <property> + <name>hbase.mob.cache.evict.remain.ratio</name> + <value>0.5f</value> + </property> + + <!--开启mob--> + <property> + <name>hfile.format.version</name> + <value>3</value> + </property> + + <property> + <name>hbase.hregion.memstore.flush.size</name> + <value>534217728</value> + </property> + <!-- flush线程数 --> + <property> + <name>hbase.hstore.flusher.count</name> + <value>6</value> + </property> + <property> + <name>hbase.regionserver.global.memstore.size.lower.limit</name> + <value>0.4</value> + </property> + + <property> + <name>hbase.regionserver.global.memstore.size</name> + <value>0.45</value> + </property> + + <property> + <name>hfile.block.cache.size</name> + <value>0.3</value> + </property> + + <property> + <name>hbase.hregion.memstore.block.multiplier</name> + <value>10</value> + </property> + + <property> + <name>hbase.ipc.server.max.callqueue.length</name> + <value>1073741824</value> + </property> + + <property> + <name>hbase.regionserver.handler.count</name> + <value>{{ regionserverhandlers }}</value> + <description>Count of RPC Listener instances spun up on RegionServers. + Same property is used by the Master for count of master handlers.</description> + </property> + + <property> + <name>hbase.zookeeper.property.maxClientCnxns</name> + <value>1000</value> + </property> + + <property> + <name>hbase.ipc.max.request.size</name> + <value>1173741824</value> + </property> + + <property> + <name>hbase.hstore.blockingWaitTime</name> + <value>30000</value> + </property> + <property> + <name>hbase.hstore.blockingStoreFiles</name> + <value>100</value> + </property> + + <!--split参数--> + <property> + <name>hbase.hregion.max.filesize</name> + <value>21474836480</value> + </property> + <property> + <name>hbase.regionserver.regionSplitLimit</name> + <value>1000</value> + </property> +</configuration> diff --git a/parcels/roles/components/hbase/templates/keephbasemaster.j2 b/parcels/roles/components/hbase/templates/keephbasemaster.j2 new file mode 100644 index 0000000..25dbe0b --- /dev/null +++ b/parcels/roles/components/hbase/templates/keephbasemaster.j2 @@ -0,0 +1,38 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 45 55 +# description: keephbasemaster +source /etc/profile + +PRO_NAME=keephbasemaster + +INS_DIR={{ data_path }} +#版本 +VERSION={{ hbase_version }} + +case $1 in +start) +master=`ps -ef | grep dae-hmaster.sh | grep -v grep | wc -l` +if [ $master -lt 1 ];then +nohup $INS_DIR/$VERSION/bin/dae-hmaster.sh > /dev/null 2>&1 & +fi +;; + +stop) +keeppid=`ps -ef | grep dae-hmaster.sh | grep -v grep | awk '{print $2}'` +echo "master守护进程PID：$keeppid" +kill -9 $keeppid + +sh $INS_DIR/$VERSION/bin/hbase-daemon.sh stop master +;; + +status) +;; + +* ) +echo "use keephbasemaster [start|stop|status]" +;; +esac + diff --git a/parcels/roles/components/hbase/templates/keephbaseregion.j2 b/parcels/roles/components/hbase/templates/keephbaseregion.j2 new file mode 100644 index 0000000..851af14 --- /dev/null +++ b/parcels/roles/components/hbase/templates/keephbaseregion.j2 @@ -0,0 +1,39 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 45 55 +# description: keephbaseregion +source /etc/profile + +PRO_NAME=keephbaseregion + +INS_DIR={{ data_path }} +#版本 +VERSION={{ hbase_version }} + +case $1 in +start) +region=`ps -ef | grep dae-hregion.sh | grep -v grep | wc -l` + +if [ $region -lt 1 ];then +nohup $INS_DIR/$VERSION/bin/dae-hregion.sh > /dev/null 2>&1 & +fi +;; + +stop) +keeppid=`ps -ef | grep dae-hregion.sh | grep -v grep | awk '{print $2}'` +echo "hregion守护进程PID：$keeppid" +kill -9 $keeppid + +sh $INS_DIR/$VERSION/bin/hbase-daemon.sh stop regionserver +;; + +status) +;; + +* ) +echo "use keephbaseregion [start|stop|status]" +;; +esac + diff --git a/parcels/roles/components/hbase/templates/set_hbase_env.sh.j2 b/parcels/roles/components/hbase/templates/set_hbase_env.sh.j2 new file mode 100644 index 0000000..c07e48b --- /dev/null +++ b/parcels/roles/components/hbase/templates/set_hbase_env.sh.j2 @@ -0,0 +1,24 @@ +#!/bin/bash + + +num=`cat /etc/profile | grep hbase | wc -l` + +if [ $num -eq "0" ];then + echo -e "\n#hbase" >> /etc/profile + echo -e "export HBASE_HOME={{ data_path }}/{{ hbase_version }}" >> /etc/profile + echo -e "export PATH=\$HBASE_HOME/bin:\$PATH" >> /etc/profile + source /etc/profile +fi + +keeppath='/etc/init.d/keephbasemaster' + if [ -x $keeppath ];then + chkconfig --add keephbasemaster + chkconfig keephbasemaster on + fi + +keeppath='/etc/init.d/keephbaseregion' + if [ -x $keeppath ];then + chkconfig --add keephbaseregion + chkconfig keephbaseregion on + fi + diff --git a/parcels/roles/components/hbase/templates/unload_hbase.sh.j2 b/parcels/roles/components/hbase/templates/unload_hbase.sh.j2 new file mode 100644 index 0000000..ed8ae9d --- /dev/null +++ b/parcels/roles/components/hbase/templates/unload_hbase.sh.j2 @@ -0,0 +1,40 @@ +#!/bin/bash +source /etc/profile + +function killService(){ +keeppath='/etc/init.d/keephbasemaster' +if [ -x $keeppath ];then +service keephbasemaster stop +chkconfig keephbasemaster off +systemctl daemon-reload +rm -rf /etc/init.d/keephbasemaster +fi + +keeppath='/etc/init.d/keephbaseregion' +if [ -x $keeppath ];then +service keephbaseregion stop +chkconfig keephbaseregion off +systemctl daemon-reload +rm -rf /etc/init.d/keephbaseregion +fi +} + +function killPid(){ +livenum=`jps -l | egrep -w "org.apache.hadoop.hbase.master.HMaster" | grep -v grep |wc -l` +if [ $livenum -ne 0 ];then +keeppid=`jps -l |egrep -w "org.apache.hadoop.hbase.master.HMaster" | awk '{print $1}'` +kill -9 $keeppid +fi + +livenum=`jps -l | egrep -w "org.apache.hadoop.hbase.regionserver.HRegionServer" | grep -v grep |wc -l` +if [ $livenum -ne 0 ];then +keeppid=`jps -l |egrep -w "org.apache.hadoop.hbase.regionserver.HRegionServer" | awk '{print $1}'` +kill -9 $keeppid +fi +} + +killService +sleep 15 +killPid +rm -rf {{ install_path }}/{{ hbase_version }} + diff --git a/parcels/roles/components/hbase/vars/main.yml b/parcels/roles/components/hbase/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/components/hbase/vars/main.yml diff --git a/parcels/roles/components/kafka/tasks/install_kafka.yml b/parcels/roles/components/kafka/tasks/install_kafka.yml new file mode 100644 index 0000000..07e97d9 --- /dev/null +++ b/parcels/roles/components/kafka/tasks/install_kafka.yml @@ -0,0 +1,236 @@ +- name: partion_nums to ansible variable + set_fact: partion_nums="{{groups.kafka|length}}" + +- name: To terminate execution + fail: + msg: "kafka 集群最少需要配置3个节点，请修改configurations/hosts内对应IP列表。" + when: partion_nums < '3' + +- name: check jdk 1.8_73 + shell: source /etc/profile && java -version 2>&1 | grep 1.8.0_73 | wc -l + ignore_errors: false + register: jdk_out + +- name: To terminate execution + fail: + msg: "检测到目标集群内存在JDK未安装节点，请检查" + when: jdk_out.stdout != '2' + run_once: true + delegate_to: 127.0.0.1 + +- name: create kafka package path:{{ install_path }} + file: + state: directory + path: '{{ install_path }}' + +- block: + - name: Check if the Kafka service already exists + shell: source /etc/profile && jps -l | grep "kafka.Kafka" | grep -v grep |awk '{print $1}' | wc -l + register: check_out + - name: copy unload_kafka.sh to {{ install_path }}/ + template: + src: 'unload_kafka.sh.j2' + dest: '{{ install_path }}/unload_kafka.sh' + force: true + mode: 0755 + when: check_out.stdout >= '1' + - name: unload kafka + shell: cd {{ install_path }} && sh unload_kafka.sh + when: check_out.stdout >= '1' + - name: Ansible delete {{ install_path }}/unload_kafka.sh + file: + path: "{{ install_path }}/unload_kafka.sh" + state: absent + - name: Check if the Kafka service already exists + shell: source /etc/profile && jps -l | grep "kafka.Kafka" | grep -v grep |awk '{print $1}' | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "卸载失败，组件可能非本安装部署，请手动卸载后继续安装" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "yes" + +- block: + - name: Check if the Kafka service already exists + shell: source /etc/profile && jps -l | grep "kafka.Kafka" | grep -v grep |awk '{print $1}' | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "检测到目标集群内已有Kafka请检查IP是否正确或停止/卸载已启动的组件" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "no" + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181" + when: '(groups.zookeeper|length) == 3' + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181,{{groups.zookeeper[3]}}:2181,{{groups.zookeeper[4]}}:2181" + when: '(groups.zookeeper|length) >= 5' + +- name: manager_ip to ansible variable + set_fact: manager_ip="{{groups.kafka[0]}}" + +- name: create kafka log path:{{ data_path }}/{{ kafka_version }}/kafka-logs + file: + state: directory + path: '{{ data_path }}/{{ kafka_version }}/kafka-logs' + +#复制tar到目标服务器 +- name: copy {{ kafka_version }}.tgz + copy: + src: '{{ package_path }}/{{ kafka_version }}.tgz' + dest: '{{ install_path }}/' + force: true + backup: yes + +#解压tar +- name: unpack {{ kafka_version }}.tgz + unarchive: + src: '{{ kafka_version }}.tgz' + dest: '{{ install_path }}/' + +#复制tar到目标服务器 +- name: copy {{ kafka_manager_version }}.zip + copy: + src: '{{ package_path }}/{{ kafka_manager_version }}.zip' + dest: '{{ install_path }}/' + force: true + run_once: true + delegate_to: "{{ manager_ip }}" + +#解压tar +- name: unpack {{ kafka_manager_version }}.zip + unarchive: + src: '{{ kafka_manager_version }}.zip' + dest: '{{ install_path }}/' + run_once: true + delegate_to: "{{ manager_ip }}" + +- name: offsets_replication to ansible variable + set_fact: offsets_replication="3" + when: partion_nums >= '3' + +- name: offsets_replication to ansible variable + set_fact: offsets_replication="1" + when: partion_nums < '3' + +#拷贝配置文件 +- name: copy server.properties + template: + src: 'server.properties.j2' + dest: '{{ install_path }}/{{ kafka_version }}/config/server.properties' + backup: yes + +#拷贝系统启动脚本 +- name: copy kafka-server-start.sh + template: + src: 'kafka-server-start.sh.j2' + dest: '{{ install_path }}/{{ kafka_version }}/bin/kafka-server-start.sh' + mode: 0755 + backup: yes + +#拷贝守护进程 +- name: copy kafka-operation.sh + template: + src: 'kafka-operation.sh.j2' + dest: '{{ install_path }}/{{ kafka_version }}/bin/kafka-operation.sh' + mode: 0755 + backup: yes + +#拷贝监控启动脚本 +- name: copy start_exporter.sh + template: + src: 'start_exporter.sh.j2' + dest: '{{ install_path }}/{{ kafka_version }}/exporter/start_exporter.sh' + mode: 0755 + backup: yes + +#拷贝开机自启守护进程 +- name: copy keepkafalive + template: + src: 'keepkafalive.j2' + dest: '/etc/init.d/keepkafalive' + mode: 0755 + backup: yes + +#拷贝守护进程 +- name: copy dae-kafka.sh + template: + src: 'dae-kafka.sh.j2' + dest: '{{ install_path }}/{{ kafka_version }}/bin/dae-kafka.sh' + mode: 0755 + backup: yes + +#拷贝日志删除脚本 +- name: copy kflogdelete.sh + template: + src: 'kflogdelete.sh.j2' + dest: '{{ install_path }}/{{ kafka_version }}/bin/kflogdelete.sh' + mode: 0755 + backup: yes + +#拷贝环境变量配置脚本 +- name: copy set_kafka_env.sh + template: + src: 'set_kafka_env.sh.j2' + dest: '{{ install_path }}/{{ kafka_version }}/bin/set_kafka_env.sh' + mode: 0755 + backup: yes + +#配置环境变量 +- name: set kafka env and chkconfig keepkafkaalive + shell: cd {{ install_path }}/{{ kafka_version }}/bin/ && ./set_kafka_env.sh + +- name: copy application.conf + template: + src: 'application.conf.j2' + dest: '{{ install_path }}/{{ kafka_manager_version }}/conf/application.conf' + backup: yes + run_once: true + delegate_to: "{{ manager_ip }}" + +- name: copy change_broker.sh + template: + src: 'change_broker.sh.j2' + dest: '~/change_broker.sh' + mode: 0755 + run_once: true + delegate_to: "127.0.0.1" + +- name: perform change_broker.sh + shell: 'sh ~/change_broker.sh' + run_once: true + delegate_to: "127.0.0.1" + +#启动kafka守护 +- name: start keepkafalive service + shell: source /etc/profile && sh /etc/init.d/keepkafalive 'start' + +- name: Ansible delete {{ kafka_version }}.tar.gz + file: + path: "{{ install_path }}/{{ kafka_version }}.tgz" + state: absent + +- name: Ansible delete {{ kafka_manager_version }}.tar.gz + file: + path: "{{ install_path }}/{{ kafka_manager_version }}.zip" + state: absent + +#导入exporter镜像 +- name: load kafka_expoter + docker_image: + name: 'kafka_exporter' + tag: 'v1.0' + load_path: '{{ install_path }}/{{ kafka_version }}/exporter/kafka_exporter.tar.gz' + source: load + force_tag: yes + force_source: yes + +#启动exporter +- name: start kafka_exporter + shell: cd {{ install_path }}/{{ kafka_version }}/exporter/ && ./start_exporter.sh diff --git a/parcels/roles/components/kafka/tasks/main.yml b/parcels/roles/components/kafka/tasks/main.yml new file mode 100644 index 0000000..abf53ed --- /dev/null +++ b/parcels/roles/components/kafka/tasks/main.yml @@ -0,0 +1 @@ +- include: install_kafka.yml diff --git a/parcels/roles/components/kafka/templates/application.conf.j2 b/parcels/roles/components/kafka/templates/application.conf.j2 new file mode 100644 index 0000000..3241631 --- /dev/null +++ b/parcels/roles/components/kafka/templates/application.conf.j2 @@ -0,0 +1,47 @@ + +# Copyright 2015 Yahoo Inc. Licensed under the Apache License, Version 2.0 +# See accompanying LICENSE file. + +# This is the main configuration file for the application. +# ~~~~~ + +# Secret key +# ~~~~~ +# The secret key is used to secure cryptographics functions. +# If you deploy your application to several instances be sure to use the same key! +play.crypto.secret="^<csmm5Fx4d=r2HEX8pelM3iBkFVv?k[mc;IZE<_Qoq8EkX_/7@Zt6dP05Pzea3U" +play.crypto.secret=${?APPLICATION_SECRET} + +# The application languages +# ~~~~~ +play.i18n.langs=["en"] + +play.http.requestHandler = "play.http.DefaultHttpRequestHandler" +play.http.context = "/" +play.application.loader=loader.KafkaManagerLoader + +kafka-manager.zkhosts="{{ zookeeper_servers }}" +pinned-dispatcher.type="PinnedDispatcher" +pinned-dispatcher.executor="thread-pool-executor" +application.features=["KMClusterManagerFeature","KMTopicManagerFeature","KMPreferredReplicaElectionFeature","KMReassignPartitionsFeature"] + +akka { + loggers = ["akka.event.slf4j.Slf4jLogger"] + loglevel = "INFO" +} + +http.port="9998" + +akka.logger-startup-timeout = 60s + +basicAuthentication.enabled=false +basicAuthentication.enabled=${?KAFKA_MANAGER_AUTH_ENABLED} +basicAuthentication.username="admin" +basicAuthentication.username=${?KAFKA_MANAGER_USERNAME} +basicAuthentication.password="password" +basicAuthentication.password=${?KAFKA_MANAGER_PASSWORD} +basicAuthentication.realm="Kafka-Manager" +basicAuthentication.excluded=["/api/health"] # ping the health of your instance without authentification + +kafka-manager.consumer.properties.file=${?CONSUMER_PROPERTIES_FILE} + diff --git a/parcels/roles/components/kafka/templates/change_broker.sh.j2 b/parcels/roles/components/kafka/templates/change_broker.sh.j2 new file mode 100755 index 0000000..7a1e8f9 --- /dev/null +++ b/parcels/roles/components/kafka/templates/change_broker.sh.j2 @@ -0,0 +1,11 @@ +#!/bin/bash + +id={{ kafka_start_broker }} + +for i in `echo "{{ ansible_play_hosts }}" | grep -E -o "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+"` +do + ip=`echo $i | grep -E -o "[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"` + ssh root@$ip sed -i 's/broker.id=0/broker.id='$id'/' {{ install_path }}/{{ kafka_version }}/config/server.properties + ((id++)) +done + diff --git a/parcels/roles/components/kafka/templates/dae-kafka.sh.j2 b/parcels/roles/components/kafka/templates/dae-kafka.sh.j2 new file mode 100755 index 0000000..75afbc3 --- /dev/null +++ b/parcels/roles/components/kafka/templates/dae-kafka.sh.j2 @@ -0,0 +1,32 @@ +#!/bin/sh + +#修改BASE_DIR为安装的路径 +#启动命令端口可自行指定 +#JMX_PORT=9191 nohup $BASE_DIR/bin/kafka-server-start.sh $BASE_DIR/config/server.properties > /dev/null 2>&1 & + +PRO_NAME=Kafka +BASE_DIR={{ install_path }} +VERSION={{ kafka_version }} +MANAGER={{ kafka_manager_version }} +source /etc/profile + + +while true ; do + NUM=`jps | grep -w ${PRO_NAME} | grep -v grep |wc -l` + + if [ "${NUM}" -lt "1" ];then + JMX_PORT=9191 nohup $BASE_DIR/$VERSION/bin/kafka-server-start.sh $BASE_DIR/$VERSION/config/server.properties > /dev/null 2>&1 & + OLD_NUM=`cat $BASE_DIR/$VERSION/logs/restart_sum.log` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASE_DIR/$VERSION/logs/restart_sum.log + echo "`date "+%Y-%m-%d %H:%M:%S"` - kafka服务启动/异常重启 - 重启次数 -> $RESTART_NUM" >> $BASE_DIR/$VERSION/restart.log + fi + + if [ -d "$BASE_DIR/$MANAGER" ];then + if [ -f "$BASE_DIR/$MANAGER/RUNNING_PID" ]; then + rm -rf $BASE_DIR/$MANAGER/RUNNING_PID + fi + nohup $BASE_DIR/$MANAGER/bin/kafka-manager -Dconfig.file=$BASE_DIR/$MANAGER/conf/application.conf > /dev/null 2>&1 & + fi +sleep 60 +done diff --git a/parcels/roles/components/kafka/templates/kafka-operation.sh.j2 b/parcels/roles/components/kafka/templates/kafka-operation.sh.j2 new file mode 100755 index 0000000..8da25d6 --- /dev/null +++ b/parcels/roles/components/kafka/templates/kafka-operation.sh.j2 @@ -0,0 +1,36 @@ +#!/bin/bash + +source /etc/profile +BASE_DIR=$KAFKA_HOME +IP_LIST={{ inventory_hostname }}:9092 +ZK_LISR={{ zookeeper_servers }}/kafka +partitions=1 +case $1 in + producer) + exec $BASE_DIR/bin/kafka-console-producer.sh '--broker-list' $IP_LIST '--topic' $2 + ;; + consumer) + exec $BASE_DIR/bin/kafka-console-consumer.sh '--bootstrap-server' $IP_LIST '--topic' $2 + ;; + consumer-begin) + exec $BASE_DIR/bin/kafka-console-consumer.sh '--bootstrap-server' $IP_LIST '--topic' $2 '--from-beginning' + ;; + create) + exec $BASE_DIR/bin/kafka-topics.sh '--create' '--zookeeper' $ZK_LISR '--replication-factor' 1 '--partitions' $partitions '--topic' $2 + ;; + list) + exec $BASE_DIR/bin/kafka-topics.sh '--list' '--zookeeper' $ZK_LISR + ;; + start) + JMX_PORT=9191 nohup $BASE_DIR/bin/kafka-server-start.sh $BASE_DIR/config/server.properties > /dev/null 2>&1 & + ;; + stop) + exec $BASE_DIR/bin/kafka-server-stop.sh + ;; + group) + exec $BASE_DIR/bin/kafka-consumer-groups.sh '--bootstrap-server' $IP_LIST '--describe' '--group' $2 + ;; + *) + echo 'Usage: kafka-operation.sh {producer|consumer|consumer-begin|create} {topic-name}' + echo 'Status: kafka-operation.sh {start|stop|list}' +esac diff --git a/parcels/roles/components/kafka/templates/kafka-server-start.sh.j2 b/parcels/roles/components/kafka/templates/kafka-server-start.sh.j2 new file mode 100755 index 0000000..a69ac8b --- /dev/null +++ b/parcels/roles/components/kafka/templates/kafka-server-start.sh.j2 @@ -0,0 +1,46 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export KAFKA_OPTS="-javaagent:{{ install_path }}/{{ kafka_version }}/monitor/jmx_prometheus_javaagent-0.12.0.jar=9901:{{ install_path }}/{{ kafka_version }}/monitor/kafka.yml" + +if [ $# -lt 1 ]; +then + echo "USAGE: $0 [-daemon] server.properties [--override property=value]*" + exit 1 +fi +base_dir=$(dirname $0) + +if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then + export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties" +fi + +if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then + export KAFKA_HEAP_OPTS="-Xmx{{ kafka_max_mem }}m -Xms{{ kafka_min_mem }}m" +fi + +EXTRA_ARGS=${EXTRA_ARGS-'-name kafkaServer -loggc'} + +COMMAND=$1 +case $COMMAND in + -daemon) + EXTRA_ARGS="-daemon "$EXTRA_ARGS + shift + ;; + *) + ;; +esac + +exec $base_dir/kafka-run-class.sh $EXTRA_ARGS kafka.Kafka "$@" diff --git a/parcels/roles/components/kafka/templates/keepkafalive.j2 b/parcels/roles/components/kafka/templates/keepkafalive.j2 new file mode 100755 index 0000000..2e2f144 --- /dev/null +++ b/parcels/roles/components/kafka/templates/keepkafalive.j2 @@ -0,0 +1,42 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 30 70 +# description: keepkafkaalive +source /etc/profile +PRO_NAME=keepkafkaalive + +case $1 in +start) +keepnum=`ps -ef | grep dae-kafka.sh | grep -v grep | wc -l` + +if [ $keepnum -eq "0" ];then +nohup {{ install_path }}/{{ kafka_version }}/bin/dae-kafka.sh > /dev/null 2>&1 & +fi +;; + +stop) +keeppid=`ps -ef | grep dae-kafka.sh | grep -v grep | awk '{print $2}'` +if [ $keeppid ];then +echo "守护进程PID：$keeppid" +kill -9 $keeppid +sleep 1 +sh {{ install_path }}/{{ kafka_version }}/bin/kafka-server-stop.sh +fi +;; + +status) +num=`jps | grep -w Kafka | grep -v grep |wc -l` +if [ "$num" -eq "1" ];then +echo "kafka进程已启动" +else +echo "kafka进程未启动" +fi +;; + +* ) +echo "use keepkafkaalive [start|stop|status]" +;; +esac + diff --git a/parcels/roles/components/kafka/templates/kflogdelete.sh.j2 b/parcels/roles/components/kafka/templates/kflogdelete.sh.j2 new file mode 100755 index 0000000..4bf5a38 --- /dev/null +++ b/parcels/roles/components/kafka/templates/kflogdelete.sh.j2 @@ -0,0 +1,16 @@ +#!/bin/sh + +#只保留最近三天的日志，如需要多保留几天修改8行最后 -n days +#将此脚本加载到系统定时任务中 /etc/crontab +#脚本会读取环境变量，固需要配置环境变量。 +#. /etc/profile + +kafka_dir={{ install_path }}/{{ kafka_version }} + +day=$(date +"%Y-%m-%d" -d "-7 days") + +kafka=`jps | grep Kafka | grep -v grep | wc -l` +if [[ $kafka = "1" ]];then + rm -rf $kafka_dir/logs/*.$day* +fi + diff --git a/parcels/roles/components/kafka/templates/server.properties.j2 b/parcels/roles/components/kafka/templates/server.properties.j2 new file mode 100644 index 0000000..be2b50e --- /dev/null +++ b/parcels/roles/components/kafka/templates/server.properties.j2 @@ -0,0 +1,153 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# see kafka.server.KafkaConfig for additional details and defaults + +############################# Server Basics ############################# + +# The id of the broker. This must be set to a unique integer for each broker. +broker.id=0 + +############################# Socket Server Settings ############################# + +# The address the socket server listens on. It will get the value returned from +# java.net.InetAddress.getCanonicalHostName() if not configured. +# FORMAT: +# listeners = listener_name://host_name:port +# EXAMPLE: +# listeners = PLAINTEXT://your.host.name:9092 +listeners=PLAINTEXT://{{ inventory_hostname }}:{{ server_port }} + +# Hostname and port the broker will advertise to producers and consumers. If not set, +# it uses the value for "listeners" if configured. Otherwise, it will use the value +# returned from java.net.InetAddress.getCanonicalHostName(). +advertised.listeners=PLAINTEXT://{{ inventory_hostname }}:{{ server_port }} + +# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details +#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL + +#kafka port +port={{ server_port }} + +#Is it deleted directlytopic +delete.topic.enable=true + +#Are you allowed to create automatically topic +auto.create.topics.enable=false + +#Enable log periodic deletion strategy +log.cleanup.policy=delete + +# The number of threads that the server uses for receiving requests from the network and sending responses to the network +num.network.threads=3 + +# The number of threads that the server uses for processing requests, which may include disk I/O +num.io.threads=8 + +# The send buffer (SO_SNDBUF) used by the socket server +socket.send.buffer.bytes=10485760 + +# The receive buffer (SO_RCVBUF) used by the socket server +socket.receive.buffer.bytes=10485760 + +# The maximum size of a request that the socket server will accept (protection against OOM) +socket.request.max.bytes=104857600 + +#The maximum size of a message body, unit byte. +message.max.bytes=10485760 + +#replicas Maximum size of data obtained eachtime +replica.fetch.max.bytes=20485760 + +############################# Log Basics ############################# + +# A comma seperated list of directories under which to store log files +log.dirs={{ data_path }}/{{ kafka_version }}/kafka-logs + +# The default number of log partitions per topic. More partitions allow greater +# parallelism for consumption, but this will also result in more files across +# the brokers. +num.partitions=1 + +# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. +# This value is recommended to be increased for installations with data dirs located in RAID array. +num.recovery.threads.per.data.dir=1 + +############################# Internal Topic Settings ############################# +# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" +# For anything other than development testing, a value greater than 1 is recommended for to ensure availability such as 3. +offsets.topic.replication.factor={{ offsets_replication }} +transaction.state.log.replication.factor=1 +transaction.state.log.min.isr=1 + +############################# Log Flush Policy ############################# + +# Messages are immediately written to the filesystem but by default we only fsync() to sync +# the OS cache lazily. The following configurations control the flush of data to disk. +# There are a few important trade-offs here: +# 1. Durability: Unflushed data may be lost if you are not using replication. +# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. +# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks. +# The settings below allow one to configure the flush policy to flush data after a period of time or +# every N messages (or both). This can be done globally and overridden on a per-topic basis. + +# The number of messages to accept before forcing a flush of data to disk +log.flush.interval.messages=20000 + +# The maximum amount of time a message can sit in a log before we force a flush +log.flush.interval.ms=2000 + +############################# Log Retention Policy ############################# + +# The following configurations control the disposal of log segments. The policy can +# be set to delete segments after a period of time, or after a given size has accumulated. +# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens +# from the end of the log. + +# The minimum age of a log file to be eligible for deletion due to age +log.retention.hours={{ log_reten_hours }} + +# A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below log.retention.bytes. Functions independently of log.retention.hours. +log.retention.bytes={{ topic_max_bytes }} + +# The maximum size of a log segment file. When this size is reached a new log segment will be created. +log.segment.bytes={{ topic_file_bytes }} + +# The interval at which log segments are checked to see if they can be deleted according +# to the retention policies +log.retention.check.interval.ms=300000 + +############################# Zookeeper ############################# + +# Zookeeper connection string (see zookeeper docs for details). +# This is a comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". +# You can also append an optional chroot string to the urls to specify the +# root directory for all kafka znodes. +zookeeper.connect={{ zookeeper_servers }}/kafka + +# Timeout in ms for connecting to zookeeper +zookeeper.connection.timeout.ms=30000 +zookeeper.session.timeout.ms=30000 + +############################# Group Coordinator Settings ############################# + +# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. +# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. +# The default value for this is 3 seconds. +# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. +# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. +group.initial.rebalance.delay.ms=0 diff --git a/parcels/roles/components/kafka/templates/set_kafka_env.sh.j2 b/parcels/roles/components/kafka/templates/set_kafka_env.sh.j2 new file mode 100755 index 0000000..755b2dc --- /dev/null +++ b/parcels/roles/components/kafka/templates/set_kafka_env.sh.j2 @@ -0,0 +1,17 @@ +#!/bin/bash + +num=`cat /etc/profile | grep kafka | wc -l` + +if [ $num -eq "0" ];then + echo -e "\n#kafka" >> /etc/profile + echo -e "export KAFKA_HOME={{ install_path }}/{{ kafka_version }}" >> /etc/profile + echo -e "export PATH=\$KAFKA_HOME/bin:\$PATH" >> /etc/profile + source /etc/profile +fi + +keeppath='/etc/init.d/keepkafalive' +if [ -x $keeppath ];then + chkconfig --add keepkafalive + chkconfig keepkafalive on +fi + diff --git a/parcels/roles/components/kafka/templates/start_exporter.sh.j2 b/parcels/roles/components/kafka/templates/start_exporter.sh.j2 new file mode 100755 index 0000000..36c0f6f --- /dev/null +++ b/parcels/roles/components/kafka/templates/start_exporter.sh.j2 @@ -0,0 +1,7 @@ +#!/bin/bash + +docker rm -f kafka_exporter + +sleep 3 + +docker run -d -p 9982:9308 --name kafka_exporter --restart always kafka_exporter:v1.0 --kafka.server={{ inventory_hostname }}:9092 diff --git a/parcels/roles/components/kafka/templates/unload_kafka.sh.j2 b/parcels/roles/components/kafka/templates/unload_kafka.sh.j2 new file mode 100755 index 0000000..1800a96 --- /dev/null +++ b/parcels/roles/components/kafka/templates/unload_kafka.sh.j2 @@ -0,0 +1,27 @@ +#!/bin/bash +source /etc/profile + +function killService(){ +keeppath='/etc/init.d/keepkafalive' +if [ -x $keeppath ];then +service keepkafalive stop +chkconfig keepkafalive off +systemctl daemon-reload +rm -rf /etc/init.d/keepkafalive +fi +} + +function killPid(){ +livenum=`jps -l | egrep -w "kafka.Kafka" | grep -v grep |wc -l` +if [ $livenum -ne 0 ];then +keeppid=`jps -l |egrep -w "kafka.Kafka" | awk '{print $1}'` +kill -9 $keeppid +fi +} + +killService +sleep 5 +killPid +rm -rf {{ install_path }}/{{ kafka_version }} +rm -rf {{ data_path }}/{{ kafka_version }} +sleep 5 diff --git a/parcels/roles/components/kafka/vars/main.yml b/parcels/roles/components/kafka/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/components/kafka/vars/main.yml diff --git a/parcels/roles/components/mariadb/install-mariadb-cluster/file/slow_query.log b/parcels/roles/components/mariadb/install-mariadb-cluster/file/slow_query.log new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/components/mariadb/install-mariadb-cluster/file/slow_query.log diff --git a/parcels/roles/components/mariadb/install-mariadb-cluster/tasks/install-mariadb.yml b/parcels/roles/components/mariadb/install-mariadb-cluster/tasks/install-mariadb.yml new file mode 100644 index 0000000..c861cf6 --- /dev/null +++ b/parcels/roles/components/mariadb/install-mariadb-cluster/tasks/install-mariadb.yml @@ -0,0 +1,116 @@ +- block: + - name: Check if the Mariadb service already exists + shell: rpm -qa | grep MariaDB | wc -l + register: check_out + - name: copy unload_mariadb.sh to {{ install_path }}/ + template: + src: 'unload_mariadb.sh.j2' + dest: '{{ install_path }}/unload_mariadb.sh' + force: true + mode: 0755 + when: check_out.stdout >= '1' + - name: unload kafka + shell: cd {{ install_path }} && sh unload_mariadb.sh + when: check_out.stdout >= '1' + - name: Ansible delete {{ install_path }}/unload_mariadb.sh + file: + path: "{{ install_path }}/unload_mariadb.sh" + state: absent + - name: Check if the Mariadb service already exists + shell: rpm -qa | grep MariaDB | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "卸载失败，组件可能非本安装部署，请手动卸载后继续安装" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "yes" + +- block: + - name: Check if the Mariadb service already exists + shell: rpm -qa | grep MariaDB | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "检测到目标集群内已有Mariadb请检查IP是否正确或停止/卸载已启动的组件" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "no" + +- name: check mariadb version; if mariadb does not exist, it will be print error;Don't Worry!!! + shell: mysql -uroot + ignore_errors: True + register: result + +- name: create directory path:/data/mariadb/logs + file: + state: directory + path: '/data/mariadb/logs' + when: result.stderr is defined and result.stderr != '' #有错误信息,说明mariadb服务不存在 + +- name: create directory path:/data/mariadb/sql + file: + state: directory + path: '/data/mariadb/sql' + when: result.stderr is defined and result.stderr != '' #有错误信息,说明mariadb服务不存在 + +- name: create directory path:{{ install_path }}/mariadb/sql + file: + state: directory + path: '{{ install_path }}/mariadb' + when: result.stderr is defined and result.stderr != '' + +- name: copy + copy: src=files/slow_query.log dest=/data/mariadb/logs force=true backup=yes + when: result.stderr is defined and result.stderr != '' #有错误信息,说明mariadb服务不存在 + +- name: mariadb service not exist; copy -rpm.zip and unzip mariadb.zip to {{ install_path }}/mariadb + unarchive: src={{ package_path }}/mariadb.zip dest={{ install_path }}/mariadb copy=yes + when: result.stderr is defined and result.stderr != '' #有错误信息,说明mariadb服务不存在 + +- name: mariadb service not exist; install mariadb + shell: rpm -ivh {{ install_path }}/mariadb/*.rpm --force --nodeps && systemctl start mysql && systemctl enable mysql + when: result.stderr is defined and result.stderr != '' #有错误信息,说明mariadb服务不存在 + register: install_mariadb_result + +- name: copy my.cnf + template: + src: my.cnf_master.j2 + dest: '/etc/my.cnf' + force: yes + run_once: true + delegate_to: '{{ groups.mariadb[0] }}' + when: result.stderr is defined and result.stderr != '' #有错误信息,说明mariadb服务不存在 + +- name: copy my.cnf + template: + src: my.cnf_slave.j2 + dest: '/etc/my.cnf' + force: yes + run_once: true + delegate_to: '{{ groups.mariadb[1] }}' + when: result.stderr is defined and result.stderr != '' #有错误信息,说明mariadb服务不存在 + +- name: change mariadb root password + shell: mysql -uroot -e "ALTER USER 'root'@'localhost' IDENTIFIED BY '{{ galaxy_mariadb_pin }}';" + when: result.stderr is defined and result.stderr != '' #有错误信息,说明mariadb服务不存在 + +- name: change mariadb remote authority + shell: mysql -uroot -p{{ galaxy_mariadb_pin }} -e"use mysql;grant all privileges on *.* to 'root'@'%' identified by '{{ galaxy_mariadb_pin }}' with grant option;FLUSH PRIVILEGES;" + when: result.stderr is defined and result.stderr != '' #有错误信息,说明mariadb服务不存在 + +- name: restart mariadb + shell: systemctl restart mysql + +- name: copy {{ install_path }}/mariadb up_mysql.sh + template: + src: up_mysql.sh.j2 + dest: '{{ install_path }}/mariadb/up_mysql.sh' + mode: 755 + backup: yes + +- name: start {{ install_path }}/mariadb up_mysql.sh + shell: cd {{ install_path }}/mariadb && ./up_mysql.sh + register: command_result diff --git a/parcels/roles/components/mariadb/install-mariadb-cluster/tasks/main.yml b/parcels/roles/components/mariadb/install-mariadb-cluster/tasks/main.yml new file mode 100644 index 0000000..1c25828 --- /dev/null +++ b/parcels/roles/components/mariadb/install-mariadb-cluster/tasks/main.yml @@ -0,0 +1 @@ +- include: install-mariadb.yml diff --git a/parcels/roles/components/mariadb/install-mariadb-cluster/templates/my.cnf_master.j2 b/parcels/roles/components/mariadb/install-mariadb-cluster/templates/my.cnf_master.j2 new file mode 100644 index 0000000..88cc932 --- /dev/null +++ b/parcels/roles/components/mariadb/install-mariadb-cluster/templates/my.cnf_master.j2 @@ -0,0 +1,193 @@ +# Example MariaDB config file for very large systems. +# +# This is for a large system with memory of 1G-2G where the system runs mainly +# MariaDB. +# +# MariaDB programs look for option files in a set of +# locations which depend on the deployment platform. +# You can copy this option file to one of those +# locations. For information about these locations, do: +# 'my_print_defaults --help' and see what is printed under +# Default options are read from the following files in the given order: +# More information at: http://dev.mysql.com/doc/mysql/en/option-files.html +# +# In this file, you can use all long options that a program supports. +# If you want to know which options a program supports, run the program +# with the "--help" option. + +# The following options will be passed to all MySQL clients +[client] +#password = your_password +port = 3306 +socket = /var/lib/mysql/mysql.sock + +# Here follows entries for some specific programs + +# The MySQL server +[mysqld] +port = 3306 +#datadir = {{ mariadb_datadir_path }} +socket = /var/lib/mysql/mysql.sock +skip-external-locking +key_buffer_size = 384M +max_allowed_packet = 1M +table_open_cache = 512 +sort_buffer_size = 2M +read_buffer_size = 2M +read_rnd_buffer_size = 8M +myisam_sort_buffer_size = 64M +thread_cache_size = 8 +query_cache_size = 32M +# Try number of CPU's*2 for thread_concurrency +thread_concurrency = 8 + +innodb_file_per_table = on #为每个表都创建分区 +skip_name_resolve = on #跳过域名解析 +lower_case_table_names = 1 #忽略表名大小写 +character-set-server = utf8mb4 #数据库编码 +character-set-client = utf8mb4 +slow_query_log +long_query_time = 0.1 +slow_query_log_file = /data/mariadb/logs/slow_query.log +innodb_log_file_size = 512M +innodb_flush_method = O_DIRECT +innodb_log_files_in_group = 3 +innodb_buffer_pool_size = {{ mariadb_innodb_buffer_pool_size }}M +innodb_buffer_pool_instances = 4 +socket = /var/lib/mysql/mysql.sock +skip-external-locking +key_buffer_size = 384M +max_allowed_packet = 128M +table_open_cache = 1024 +sort_buffer_size = 2M +read_buffer_size = 2M +read_rnd_buffer_size = 8M +myisam_sort_buffer_size = 64M +thread_cache_size = 8 +query_cache_size = 32M +thread_concurrency = 16 #线程数量，推荐为cpu和核数的2倍 +bulk_insert_buffer_size=100M + +slave_skip_errors = 1062 + +# Point the following paths to a dedicated disk +#tmpdir = /tmp/ + +# Don't listen on a TCP/IP port at all. This can be a security enhancement, +# if all processes that need to connect to mysqld run on the same host. +# All interaction with mysqld must be made via Unix sockets or named pipes. +# Note that using this option without enabling named pipes on Windows +# (via the "enable-named-pipe" option) will render mysqld useless! +# +#skip-networking + +# Replication Master Server (default) +# binary logging is required for replication +log-bin=mysql-bin + +# required unique id between 1 and 2^32 - 1 +# defaults to 1 if master-host is not set +# but will not function as a master if omitted + +server-id = 1 +auto_increment_increment = {{ groups.mariadb|length }} #步进值auto_imcrement。一般有n台主MySQL就填n +auto_increment_offset = 1 #起始值。一般填第n台主MySQL。此时为第一台主MySQL + +log_slave_updates=on + +binlog-ignore-db = mysql +binlog-ignore-db = performance_schema +binlog-ignore-db = information_schema + +# Replication Slave (comment out master section to use this) +# +# To configure this host as a replication slave, you can choose between +# two methods : +# +# 1) Use the CHANGE MASTER TO command (fully described in our manual) - +# the syntax is: +# +# CHANGE MASTER TO MASTER_HOST=<host>, MASTER_PORT=<port>, +# MASTER_USER=<user>, MASTER_PASSWORD=<password> ; +# +# where you replace <host>, <user>, <password> by quoted strings and +# <port> by the master's port number (3306 by default). +# +# Example: +# +# CHANGE MASTER TO MASTER_HOST='125.564.12.1', MASTER_PORT=3306, +# MASTER_USER='joe', MASTER_PASSWORD='secret'; +# +# OR +# +# 2) Set the variables below. However, in case you choose this method, then +# start replication for the first time (even unsuccessfully, for example +# if you mistyped the password in master-password and the slave fails to +# connect), the slave will create a master.info file, and any later +# change in this file to the variables' values below will be ignored and +# overridden by the content of the master.info file, unless you shutdown +# the slave server, delete master.info and restart the slaver server. +# For that reason, you may want to leave the lines below untouched +# (commented) and instead use CHANGE MASTER TO (see above) +# +# required unique id between 2 and 2^32 - 1 +# (and different from the master) +# defaults to 2 if master-host is set +# but will not function as a slave if omitted +#server-id = 2 +# +# The replication master for this slave - required +#master-host = <hostname> +# +# The username the slave will use for authentication when connecting +# to the master - required +#master-user = <username> +# +# The password the slave will authenticate with when connecting to +# the master - required +#master-password = <password> +# +# The port the master is listening on. +# optional - defaults to 3306 +#master-port = <port> +# +# binary logging - not required for slaves, but recommended +#log-bin=mysql-bin +# +# binary logging format - mixed recommended +#binlog_format=mixed + +# Uncomment the following if you are using InnoDB tables +#innodb_data_home_dir = /usr/local/mysql/data +#innodb_data_file_path = ibdata1:2000M;ibdata2:10M:autoextend +#innodb_log_group_home_dir = /usr/local/mysql/data +# You can set .._buffer_pool_size up to 50 - 80 % +# of RAM but beware of setting memory usage too high +#innodb_buffer_pool_size = 384M +# Set .._log_file_size to 25 % of buffer pool size +#innodb_log_file_size = 100M +#innodb_log_buffer_size = 8M +#innodb_flush_log_at_trx_commit = 1 +#innodb_lock_wait_timeout = 50 + +relay-log=relay-bin +relay-log-index=relay-bin-index +slave-skip-errors=all + +[mysqldump] +quick +max_allowed_packet = 256M + +[mysql] +no-auto-rehash +# Remove the next comment character if you are not familiar with SQL +#safe-updates + +[myisamchk] +key_buffer_size = 256M +sort_buffer_size = 256M +read_buffer = 2M +write_buffer = 2M + +[mysqlhotcopy] +interactive-timeout diff --git a/parcels/roles/components/mariadb/install-mariadb-cluster/templates/my.cnf_slave.j2 b/parcels/roles/components/mariadb/install-mariadb-cluster/templates/my.cnf_slave.j2 new file mode 100644 index 0000000..40201c6 --- /dev/null +++ b/parcels/roles/components/mariadb/install-mariadb-cluster/templates/my.cnf_slave.j2 @@ -0,0 +1,193 @@ +# Example MariaDB config file for very large systems. +# +# This is for a large system with memory of 1G-2G where the system runs mainly +# MariaDB. +# +# MariaDB programs look for option files in a set of +# locations which depend on the deployment platform. +# You can copy this option file to one of those +# locations. For information about these locations, do: +# 'my_print_defaults --help' and see what is printed under +# Default options are read from the following files in the given order: +# More information at: http://dev.mysql.com/doc/mysql/en/option-files.html +# +# In this file, you can use all long options that a program supports. +# If you want to know which options a program supports, run the program +# with the "--help" option. + +# The following options will be passed to all MySQL clients +[client] +#password = your_password +port = 3306 +socket = /var/lib/mysql/mysql.sock + +# Here follows entries for some specific programs + +# The MySQL server +[mysqld] +port = 3306 +#datadir = {{ mariadb_datadir_path }} +socket = /var/lib/mysql/mysql.sock +skip-external-locking +key_buffer_size = 384M +max_allowed_packet = 1M +table_open_cache = 512 +sort_buffer_size = 2M +read_buffer_size = 2M +read_rnd_buffer_size = 8M +myisam_sort_buffer_size = 64M +thread_cache_size = 8 +query_cache_size = 32M +# Try number of CPU's*2 for thread_concurrency +thread_concurrency = 8 + +innodb_file_per_table = on #为每个表都创建分区 +skip_name_resolve = on #跳过域名解析 +lower_case_table_names = 1 #忽略表名大小写 +character-set-server = utf8mb4 #数据库编码 +character-set-client = utf8mb4 +slow_query_log +long_query_time = 0.1 +slow_query_log_file = /data/mariadb/logs/slow_query.log +innodb_log_file_size = 512M +innodb_flush_method = O_DIRECT +innodb_log_files_in_group = 3 +innodb_buffer_pool_size = {{ mariadb_innodb_buffer_pool_size }}M +innodb_buffer_pool_instances = 4 +socket = /var/lib/mysql/mysql.sock +skip-external-locking +key_buffer_size = 384M +max_allowed_packet = 128M +table_open_cache = 1024 +sort_buffer_size = 2M +read_buffer_size = 2M +read_rnd_buffer_size = 8M +myisam_sort_buffer_size = 64M +thread_cache_size = 8 +query_cache_size = 32M +thread_concurrency = 16 #线程数量，推荐为cpu和核数的2倍 +bulk_insert_buffer_size=100M + +slave_skip_errors = 1062 + +# Point the following paths to a dedicated disk +#tmpdir = /tmp/ + +# Don't listen on a TCP/IP port at all. This can be a security enhancement, +# if all processes that need to connect to mysqld run on the same host. +# All interaction with mysqld must be made via Unix sockets or named pipes. +# Note that using this option without enabling named pipes on Windows +# (via the "enable-named-pipe" option) will render mysqld useless! +# +#skip-networking + +# Replication Master Server (default) +# binary logging is required for replication +log-bin=mysql-bin + +# required unique id between 1 and 2^32 - 1 +# defaults to 1 if master-host is not set +# but will not function as a master if omitted + +server-id = 2 +auto_increment_increment = {{ groups.mariadb|length }} #步进值auto_imcrement。一般有n台主MySQL就填n +auto_increment_offset = 2 #起始值。一般填第n台主MySQL。此时为第一台主MySQL + +log_slave_updates=on + +binlog-ignore-db = mysql +binlog-ignore-db = performance_schema +binlog-ignore-db = information_schema + +# Replication Slave (comment out master section to use this) +# +# To configure this host as a replication slave, you can choose between +# two methods : +# +# 1) Use the CHANGE MASTER TO command (fully described in our manual) - +# the syntax is: +# +# CHANGE MASTER TO MASTER_HOST=<host>, MASTER_PORT=<port>, +# MASTER_USER=<user>, MASTER_PASSWORD=<password> ; +# +# where you replace <host>, <user>, <password> by quoted strings and +# <port> by the master's port number (3306 by default). +# +# Example: +# +# CHANGE MASTER TO MASTER_HOST='125.564.12.1', MASTER_PORT=3306, +# MASTER_USER='joe', MASTER_PASSWORD='secret'; +# +# OR +# +# 2) Set the variables below. However, in case you choose this method, then +# start replication for the first time (even unsuccessfully, for example +# if you mistyped the password in master-password and the slave fails to +# connect), the slave will create a master.info file, and any later +# change in this file to the variables' values below will be ignored and +# overridden by the content of the master.info file, unless you shutdown +# the slave server, delete master.info and restart the slaver server. +# For that reason, you may want to leave the lines below untouched +# (commented) and instead use CHANGE MASTER TO (see above) +# +# required unique id between 2 and 2^32 - 1 +# (and different from the master) +# defaults to 2 if master-host is set +# but will not function as a slave if omitted +#server-id = 2 +# +# The replication master for this slave - required +#master-host = <hostname> +# +# The username the slave will use for authentication when connecting +# to the master - required +#master-user = <username> +# +# The password the slave will authenticate with when connecting to +# the master - required +#master-password = <password> +# +# The port the master is listening on. +# optional - defaults to 3306 +#master-port = <port> +# +# binary logging - not required for slaves, but recommended +#log-bin=mysql-bin +# +# binary logging format - mixed recommended +#binlog_format=mixed + +# Uncomment the following if you are using InnoDB tables +#innodb_data_home_dir = /usr/local/mysql/data +#innodb_data_file_path = ibdata1:2000M;ibdata2:10M:autoextend +#innodb_log_group_home_dir = /usr/local/mysql/data +# You can set .._buffer_pool_size up to 50 - 80 % +# of RAM but beware of setting memory usage too high +#innodb_buffer_pool_size = 384M +# Set .._log_file_size to 25 % of buffer pool size +#innodb_log_file_size = 100M +#innodb_log_buffer_size = 8M +#innodb_flush_log_at_trx_commit = 1 +#innodb_lock_wait_timeout = 50 + +relay-log=relay-bin +relay-log-index=relay-bin-index +slave-skip-errors=all + +[mysqldump] +quick +max_allowed_packet = 256M + +[mysql] +no-auto-rehash +# Remove the next comment character if you are not familiar with SQL +#safe-updates + +[myisamchk] +key_buffer_size = 256M +sort_buffer_size = 256M +read_buffer = 2M +write_buffer = 2M + +[mysqlhotcopy] +interactive-timeout diff --git a/parcels/roles/components/mariadb/install-mariadb-cluster/templates/unload_mariadb.sh.j2 b/parcels/roles/components/mariadb/install-mariadb-cluster/templates/unload_mariadb.sh.j2 new file mode 100644 index 0000000..f7f9545 --- /dev/null +++ b/parcels/roles/components/mariadb/install-mariadb-cluster/templates/unload_mariadb.sh.j2 @@ -0,0 +1,23 @@ +#!/bin/bash +source /etc/profile + +function killPid(){ + +livenum=`rpm -qa | grep MariaDB | wc -l` +if [ $livenum -ne 0 ];then +systemctl stop mysql +rpm -e --nodeps MariaDB-client-10.5.3-1.el7.centos.x86_64 +rpm -e --nodeps MariaDB-shared-10.5.3-1.el7.centos.x86_64 +rpm -e --nodeps MariaDB-test-10.5.3-1.el7.centos.x86_64 +rpm -e --nodeps MariaDB-server-10.5.3-1.el7.centos.x86_64 +rpm -e --nodeps MariaDB-devel-10.5.3-1.el7.centos.x86_64 +rpm -e --nodeps MariaDB-common-10.5.3-1.el7.centos.x86_64 +rm -rf /etc/my.cnf +rm -rf /etc/my.cnf.rpmsave +rm -rf /var/lib/mysql/ +rm -rf /data/mariadb +fi + +} + +killPid diff --git a/parcels/roles/components/mariadb/install-mariadb-cluster/templates/up_mysql.sh.j2 b/parcels/roles/components/mariadb/install-mariadb-cluster/templates/up_mysql.sh.j2 new file mode 100644 index 0000000..09abb33 --- /dev/null +++ b/parcels/roles/components/mariadb/install-mariadb-cluster/templates/up_mysql.sh.j2 @@ -0,0 +1,29 @@ +#!/bin/bash +datadir_path={{ mariadb_datadir_path }}/mydata +my_conf=/etc/my.cnf + +if [ ! -d "$datadir_path" ];then + +mkdir -p $datadir_path +chown -R mysql:mysql $datadir_path +chmod 777 $datadir_path + +chown -R mysql:mysql $datadir_path +chmod 777 $datadir_path + +systemctl stop mysql + +if [ -f "$my_conf" ];then +sed -i "s%#datadir.*%datadir = $datadir_path %g" $my_conf +sed -i "s%socket.*mysql.sock%socket = $datadir_path/mysql.sock %g" $my_conf +sed -i 's/\r$//' $my_conf +fi + +if [ -d "/var/lib/mysql" ];then +cp -a /var/lib/mysql/* $datadir_path +fi +systemctl start mysql +fi + + + diff --git a/parcels/roles/components/mariadb/install-mariadb-cluster/vars/main.yml b/parcels/roles/components/mariadb/install-mariadb-cluster/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/components/mariadb/install-mariadb-cluster/vars/main.yml diff --git a/parcels/roles/components/mariadb/mariadb-double-master/file/pyMysql.zip b/parcels/roles/components/mariadb/mariadb-double-master/file/pyMysql.zip Binary files differnew file mode 100644 index 0000000..ce72a5b --- /dev/null +++ b/parcels/roles/components/mariadb/mariadb-double-master/file/pyMysql.zip diff --git a/parcels/roles/components/mariadb/mariadb-double-master/tasks/change_slave_to_master.yml b/parcels/roles/components/mariadb/mariadb-double-master/tasks/change_slave_to_master.yml new file mode 100644 index 0000000..787f8f7 --- /dev/null +++ b/parcels/roles/components/mariadb/mariadb-double-master/tasks/change_slave_to_master.yml @@ -0,0 +1,108 @@ +- name: copy pyMysql.zip + unarchive: src=file/pyMysql.zip dest=~ copy=yes + register: copy_info + +- name: pip install pyMysql + shell: cd ~/pyMysql && pip install --ignore-installed * + when: copy_info.changed + + +- name: Get mariadb slave_status + mysql_info: + login_host: '{{ groups.mariadb[0] }}' + login_user: root + login_password: '{{ galaxy_mariadb_pin }}' + filter: slave_status + register: mariadb_info + +#- name: print +# debug: +# msg: mysqlinfo is {{ mariadb_info.slave_status|length }} +# when: mariadb_info.slave_status|length == 0 + +- name: change_slave_to_master1 + mysql_replication: + login_unix_socket: "{{ mariadb_datadir_path }}/mydata/mysql.sock" + login_host: "{{ groups.mariadb[1] }}" + login_port: "{{ galaxy_mariadb_port }}" + login_user: root + login_password: "{{ galaxy_mariadb_pin }}" + master_host: "{{ groups.mariadb[0] }}" + master_user: root + master_password: "{{ galaxy_mariadb_pin }}" + master_port: "{{ galaxy_mariadb_port }}" + mode: changemaster + run_once: true + delegate_to: '{{ groups.mariadb[1] }}' +# when: slave is defined and mariadb_info.slave_status|length == 0 + tags: + - change_slave_to_master1 + + +- name: start_slave1 + mysql_replication: + login_unix_socket: "{{ mariadb_datadir_path }}/mydata/mysql.sock" + login_user: root + login_host: "{{ inventory_hostname }}" + login_port: "{{ galaxy_mariadb_port }}" + login_password: "{{ galaxy_mariadb_pin }}" + mode: startslave + tags: + - start_slave1 + run_once: true + delegate_to: '{{ groups.mariadb[1] }}' +# when: slave is defined and mariadb_info.slave_status|length == 0 + + + +- name: change_slave_to_master2 + mysql_replication: + login_unix_socket: "{{ mariadb_datadir_path }}/mydata/mysql.sock" + login_host: "{{ groups.mariadb[0] }}" + login_port: "{{ galaxy_mariadb_port }}" + login_user: root + login_password: "{{ galaxy_mariadb_pin }}" + master_host: "{{ groups.mariadb[1] }}" + master_user: root + master_password: "{{ galaxy_mariadb_pin }}" + master_port: "{{ galaxy_mariadb_port }}" + mode: changemaster + run_once: true + delegate_to: '{{ groups.mariadb[0] }}' +# when: master is defined and mariadb_info.slave_status|length == 0 + tags: + - change_slave_to_master2 + + +- name: start_slave2 + mysql_replication: + login_unix_socket: "{{ mariadb_datadir_path }}/mydata/mysql.sock" + login_user: root + login_host: "{{ inventory_hostname }}" + login_port: "{{ galaxy_mariadb_port }}" + login_password: "{{ galaxy_mariadb_pin }}" + mode: startslave + tags: + - start_slave2 + run_once: true + delegate_to: '{{ groups.mariadb[0] }}' +# when: master is defined and mariadb_info.slave_status|length == 0 + +- name: get_slave_info + mysql_replication: + login_host: 127.0.0.1 + login_user: root + login_port: "{{ galaxy_mariadb_port }}" + login_password: "{{ galaxy_mariadb_pin }}" + mode: getslave + register: info + tags: + - get_slave_info + + +#- name: dispaly_slave +# debug: +# msg: "Slave_IO_Running={{ info.Slave_IO_Running }} Slave_SQL_Running={{ info.Slave_SQL_Running }}" +# tags: +# - dispaly_slave + diff --git a/parcels/roles/components/mariadb/mariadb-double-master/tasks/change_slave_to_master.yml_bak b/parcels/roles/components/mariadb/mariadb-double-master/tasks/change_slave_to_master.yml_bak new file mode 100644 index 0000000..2e0d930 --- /dev/null +++ b/parcels/roles/components/mariadb/mariadb-double-master/tasks/change_slave_to_master.yml_bak @@ -0,0 +1,99 @@ +- name: copy pyMysql.zip + unarchive: src=file/pyMysql.zip dest=~ copy=yes + register: copy_info + +- name: pip install pyMysql + shell: cd ~/pyMysql && pip install --ignore-installed * + when: copy_info.changed + + +- name: Get mariadb slave_status + mysql_info: + login_host: '{{ groups.mariadb_master_ip[0] }}' + login_user: root + login_password: '{{ mariadb_root_pwd }}' + filter: slave_status + register: mariadb_info + +- name: print + debug: + msg: mysqlinfo is {{ mariadb_info.slave_status|length }} + when: mariadb_info.slave_status|length == 0 + +- name: change_slave_to_master1 + mysql_replication: + login_unix_socket: /data/mydb/mysql.sock + login_host: "{{ groups.mariadb_slave_ip[0] }}" + login_port: "{{ mariadb_port }}" + login_user: root + login_password: "{{ mariadb_root_pwd }}" + master_host: "{{ groups.mariadb_master_ip[0] }}" + master_user: root + master_password: "{{ mariadb_root_pwd }}" + master_port: "{{ mariadb_port }}" + mode: changemaster + when: slave is defined and mariadb_info.slave_status|length == 0 + tags: + - change_slave_to_master1 + + +- name: start_slave1 + mysql_replication: + login_unix_socket: /data/mydb/mysql.sock + login_user: root + login_host: "{{ inventory_hostname }}" + login_port: "{{ mariadb_port }}" + login_password: "{{ mariadb_root_pwd }}" + mode: startslave + tags: + - start_slave1 + when: slave is defined and mariadb_info.slave_status|length == 0 + + + +- name: change_slave_to_master2 + mysql_replication: + login_unix_socket: /data/mydb/mysql.sock + login_host: "{{ groups.mariadb_master_ip[0] }}" + login_port: "{{ mariadb_port }}" + login_user: root + login_password: "{{ mariadb_root_pwd }}" + master_host: "{{ groups.mariadb_slave_ip[0] }}" + master_user: root + master_password: "{{ mariadb_root_pwd }}" + master_port: "{{ mariadb_port }}" + mode: changemaster + when: master is defined and mariadb_info.slave_status|length == 0 + tags: + - change_slave_to_master2 + + +- name: start_slave2 + mysql_replication: + login_unix_socket: /data/mydb/mysql.sock + login_user: root + login_host: "{{ inventory_hostname }}" + login_port: "{{ mariadb_port }}" + login_password: "{{ mariadb_root_pwd }}" + mode: startslave + tags: + - start_slave2 + when: master is defined and mariadb_info.slave_status|length == 0 + +- name: get_slave_info + mysql_replication: + login_host: 127.0.0.1 + login_user: root + login_port: "{{ mariadb_port }}" + login_password: "{{ mariadb_root_pwd }}" + mode: getslave + register: info + tags: + - get_slave_info + + +- name: dispaly_slave + debug: + msg: "Slave_IO_Running={{ info.Slave_IO_Running }} Slave_SQL_Running={{ info.Slave_SQL_Running }}" + tags: + - dispaly_slave diff --git a/parcels/roles/components/mariadb/mariadb-double-master/tasks/main.yml b/parcels/roles/components/mariadb/mariadb-double-master/tasks/main.yml new file mode 100644 index 0000000..c2468d2 --- /dev/null +++ b/parcels/roles/components/mariadb/mariadb-double-master/tasks/main.yml @@ -0,0 +1,2 @@ +- include: change_slave_to_master.yml + diff --git a/parcels/roles/components/nacos/files/mysql b/parcels/roles/components/nacos/files/mysql Binary files differnew file mode 100755 index 0000000..66af1a1 --- /dev/null +++ b/parcels/roles/components/nacos/files/mysql diff --git a/parcels/roles/components/nacos/tasks/install_nacos.yml b/parcels/roles/components/nacos/tasks/install_nacos.yml new file mode 100644 index 0000000..feb4e2e --- /dev/null +++ b/parcels/roles/components/nacos/tasks/install_nacos.yml @@ -0,0 +1,166 @@ +- name: check jdk 1.8_73 + shell: source /etc/profile && java -version 2>&1 | grep 1.8.0_73 | wc -l + ignore_errors: false + register: jdk_out + +- name: To terminate execution + fail: + msg: "检测到目标集群内存在JDK未安装节点，请检查" + when: jdk_out.stdout != '2' + run_once: true + delegate_to: 127.0.0.1 + +- name: copy mysql to /usr/bin/ + copy: + src: '../files/mysql' + dest: '/usr/bin/' + force: true + +- name: change mode + shell: chmod +x /usr/bin/mysql + +- block: + - name: Check if the Nacos service already exists + shell: source /etc/profile && jps -l | grep "nacos-server" | grep -v grep |awk '{print $1}' | wc -l + register: check_out + - name: copy unload_nacos.sh to {{ install_path }}/ + template: + src: 'unload_nacos.sh.j2' + dest: '{{ install_path }}/unload_nacos.sh' + force: true + mode: 0755 + when: check_out.stdout >= '1' + - name: unload nacos + shell: cd {{ install_path }} && sh unload_nacos.sh + when: check_out.stdout >= '1' + - name: Ansible delete {{ install_path }}/unload_nacos.sh + file: + path: "{{ install_path }}/unload_nacos.sh" + state: absent + - name: Check if the Nacos service already exists + shell: source /etc/profile && jps -l | grep "nacos-server" | grep -v grep |awk '{print $1}' | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "卸载失败，组件可能非本安装部署，请手动卸载后继续安装" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "yes" + +- block: + - name: Check if the Nacos service already exists + shell: source /etc/profile && jps -l | grep "nacos-server" | grep -v grep |awk '{print $1}' | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "检测到目标集群内已有Nacos请检查IP是否正确或停止/卸载已启动的组件" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "no" + + +#创建工作目录 +- name: create nacos config directory + file: + state: directory + path: '{{ install_path }}/{{ nacos_version }}' + +#复制tar到目标服务器 +- name: copy {{ nacos_version }}.tar.gz + copy: + src: '{{ package_path }}/{{ nacos_version }}.tar.gz' + dest: '{{ install_path }}/' + force: true + +#解压tar +- name: unpack {{ nacos_version }}.tar + unarchive: + src: '{{ nacos_version }}.tar.gz' + dest: '{{ install_path }}/' +#拷贝配置文件 +- name: copy application.properties + template: + src: 'application.properties.j2' + dest: '{{ install_path }}/{{ nacos_version }}/conf/application.properties' + +- name: copy cluster.conf + template: + src: 'cluster.conf.j2' + dest: '{{ install_path }}/{{ nacos_version }}/conf/cluster.conf' + +- name: change cluster.conf + shell: 'for i in {{ ansible_play_hosts }} ;do echo `echo $i | grep -E -o "[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"`:{{ nacos_port }} >> {{ install_path }}/{{ nacos_version }}/conf/cluster.conf ;done' + +- name: copy nacos-logback.xml + template: + src: 'nacos-logback.xml.j2' + dest: '{{ install_path }}/{{ nacos_version }}/conf/nacos-logback.xml' + +- name: copy init sql nacos database + template: + src: 'nacos-mysql.sql.j2' + dest: '{{ install_path }}/{{ nacos_version }}/conf/nacos-mysql.sql' + +- name: check if {{ mariadb_nacos_database }} exist + shell: mysql -s -uroot -p{{ galaxy_mariadb_pin }} -P3306 -h{{ gateway_keepalive_host }} -e "select count(1) from information_schema.schemata where schema_name='{{ mariadb_nacos_database }}'" | grep 1 | wc -l + register: has_nacos_out + run_once: true + delegate_to: "{{groups.nacos[0]}}" + +- name: create nacos database + shell: mysql -u{{ galaxy_mariadb_username }} -p{{ galaxy_mariadb_pin }} -P{{ galaxy_mariadb_port }} -h{{ gateway_keepalive_host }} -e "drop database {{ mariadb_nacos_database }};" + run_once: true + delegate_to: '{{groups.nacos[0]}}' + when: has_nacos_out.stdout == '1' + +- name: create nacos database + shell: mysql -u{{ galaxy_mariadb_username }} -p{{ galaxy_mariadb_pin }} -P{{ galaxy_mariadb_port }} -h{{ gateway_keepalive_host }} -e "create database if not exists {{ mariadb_nacos_database }} default character set utf8mb4 collate utf8mb4_general_ci;" + run_once: true + delegate_to: '{{groups.nacos[0]}}' + +- name: exec nacos-mysql.sql + shell: 'mysql -u{{ galaxy_mariadb_username }} -p{{ galaxy_mariadb_pin }} -P{{ galaxy_mariadb_port }} -h{{ gateway_keepalive_host }} {{ mariadb_nacos_database }} < {{ install_path }}/{{ nacos_version }}/conf/nacos-mysql.sql' + run_once: true + delegate_to: '{{groups.nacos[0]}}' + +- name: copy set-nacos-env.sh + template: + src: 'set-nacos-env.sh.j2' + dest: '{{ install_path }}/{{ nacos_version }}/bin/set-nacos-env.sh' + mode: 0755 + +- name: copy startup.sh + template: + src: 'startup.sh.j2' + dest: '{{ install_path }}/{{ nacos_version }}/bin/startup.sh.j2' + mode: 0755 + +#拷贝开机自启守护进程 +- name: copy keepnacosalive + template: + src: 'keepnacosalive.j2' + dest: '/etc/init.d/keepnacosalive' + mode: 0755 + +#拷贝守护进程 +- name: copy dae-nacos.sh + template: + src: 'dae-nacos.sh.j2' + dest: '{{ install_path }}/{{ nacos_version }}/bin/dae-nacos.sh' + mode: 0755 + +- name: set nacos env and chkconfig keepnacosalive + shell: cd {{ install_path }}/{{ nacos_version }}/bin/ && ./set-nacos-env.sh + +#启动守护 +- name: start keepnacosalive service + shell: source /etc/profile && sh /etc/init.d/keepnacosalive 'start' + +- name: Ansible delete {{ nacos_version }}.tar.gz + file: + path: "{{ install_path }}/{{ nacos_version }}.tar.gz" + state: absent + + diff --git a/parcels/roles/components/nacos/tasks/main.yml b/parcels/roles/components/nacos/tasks/main.yml new file mode 100644 index 0000000..1988168 --- /dev/null +++ b/parcels/roles/components/nacos/tasks/main.yml @@ -0,0 +1 @@ +- include: install_nacos.yml diff --git a/parcels/roles/components/nacos/templates/application.properties.j2 b/parcels/roles/components/nacos/templates/application.properties.j2 new file mode 100644 index 0000000..e161a75 --- /dev/null +++ b/parcels/roles/components/nacos/templates/application.properties.j2 @@ -0,0 +1,228 @@ +# +# Copyright 1999-2018 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#*************** Spring Boot Related Configurations ***************# +### Default web context path: +server.servlet.contextPath=/nacos +### Default web server port: +server.port={{ nacos_port }} + +#*************** Network Related Configurations ***************# +### If prefer hostname over ip for Nacos server addresses in cluster.conf: +# nacos.inetutils.prefer-hostname-over-ip=false + +### Specify local server's IP: +# nacos.inetutils.ip-address= + + +#*************** Config Module Related Configurations ***************# +### If use MySQL as datasource: +# spring.datasource.platform=mysql + +### Count of DB: +# db.num=1 + +### Connect URL of DB: +db.url.0=jdbc:mysql://{{ gateway_keepalive_host }}:{{ galaxy_mariadb_port }}/{{ mariadb_nacos_database }}?characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useUnicode=true&useSSL=false&serverTimezone=UTC +db.user.0={{ galaxy_mariadb_username }} +db.password.0={{ galaxy_mariadb_pin }} + +### Connection pool configuration: hikariCP +db.pool.config.connectionTimeout=30000 +db.pool.config.validationTimeout=10000 +db.pool.config.maximumPoolSize=20 +db.pool.config.minimumIdle=2 + +#*************** Naming Module Related Configurations ***************# +### Data dispatch task execution period in milliseconds: Will removed on v2.1.X, replace with nacos.core.protocol.distro.data.sync.delayMs +# nacos.naming.distro.taskDispatchPeriod=200 + +### Data count of batch sync task: Will removed on v2.1.X. Deprecated +# nacos.naming.distro.batchSyncKeyCount=1000 + +### Retry delay in milliseconds if sync task failed: Will removed on v2.1.X, replace with nacos.core.protocol.distro.data.sync.retryDelayMs +# nacos.naming.distro.syncRetryDelay=5000 + +### If enable data warmup. If set to false, the server would accept request without local data preparation: +# nacos.naming.data.warmup=true + +### If enable the instance auto expiration, kind like of health check of instance: +# nacos.naming.expireInstance=true + +### will be removed and replaced by `nacos.naming.clean` properties +nacos.naming.empty-service.auto-clean=true +nacos.naming.empty-service.clean.initial-delay-ms=50000 +nacos.naming.empty-service.clean.period-time-ms=30000 + +### Add in 2.0.0 +### The interval to clean empty service, unit: milliseconds. +# nacos.naming.clean.empty-service.interval=60000 + +### The expired time to clean empty service, unit: milliseconds. +# nacos.naming.clean.empty-service.expired-time=60000 + +### The interval to clean expired metadata, unit: milliseconds. +# nacos.naming.clean.expired-metadata.interval=5000 + +### The expired time to clean metadata, unit: milliseconds. +# nacos.naming.clean.expired-metadata.expired-time=60000 + +### The delay time before push task to execute from service changed, unit: milliseconds. +# nacos.naming.push.pushTaskDelay=500 + +### The timeout for push task execute, unit: milliseconds. +# nacos.naming.push.pushTaskTimeout=5000 + +### The delay time for retrying failed push task, unit: milliseconds. +# nacos.naming.push.pushTaskRetryDelay=1000 + +#*************** CMDB Module Related Configurations ***************# +### The interval to dump external CMDB in seconds: +# nacos.cmdb.dumpTaskInterval=3600 + +### The interval of polling data change event in seconds: +# nacos.cmdb.eventTaskInterval=10 + +### The interval of loading labels in seconds: +# nacos.cmdb.labelTaskInterval=300 + +### If turn on data loading task: +# nacos.cmdb.loadDataAtStart=false + + +#*************** Metrics Related Configurations ***************# +### Metrics for prometheus +management.endpoints.web.exposure.include=* + +### Metrics for elastic search +management.metrics.export.elastic.enabled=false +#management.metrics.export.elastic.host=http://localhost:9200 + +### Metrics for influx +management.metrics.export.influx.enabled=false +#management.metrics.export.influx.db=springboot +#management.metrics.export.influx.uri=http://localhost:8086 +#management.metrics.export.influx.auto-create-db=true +#management.metrics.export.influx.consistency=one +#management.metrics.export.influx.compressed=true + +#*************** Access Log Related Configurations ***************# +### If turn on the access log: +server.tomcat.accesslog.enabled=true + +### The access log pattern: +server.tomcat.accesslog.pattern=%h %l %u %t "%r" %s %b %D %{User-Agent}i %{Request-Source}i + +### The directory of access log: +server.tomcat.basedir= + +#*************** Access Control Related Configurations ***************# +### If enable spring security, this option is deprecated in 1.2.0: +#spring.security.enabled=false + +### The ignore urls of auth, is deprecated in 1.2.0: +nacos.security.ignore.urls=/,/error,/**/*.css,/**/*.js,/**/*.html,/**/*.map,/**/*.svg,/**/*.png,/**/*.ico,/console-ui/public/**,/v1/auth/**,/v1/console/health/**,/actuator/**,/v1/console/server/** + +### The auth system to use, currently only 'nacos' and 'ldap' is supported: +nacos.core.auth.system.type=nacos + +### If turn on auth system: +nacos.core.auth.enabled=true + +### worked when nacos.core.auth.system.type=ldap，{0} is Placeholder,replace login username +# nacos.core.auth.ldap.url=ldap://localhost:389 +# nacos.core.auth.ldap.userdn=cn={0},ou=user,dc=company,dc=com + +### The token expiration in seconds: +nacos.core.auth.default.token.expire.seconds=18000 + +### The default token: +nacos.core.auth.default.token.secret.key=SecretKey012345678901234567890123456789012345678901234567890123456789 + +### Turn on/off caching of auth information. By turning on this switch, the update of auth information would have a 15 seconds delay. +nacos.core.auth.caching.enabled=true + +### Since 1.4.1, Turn on/off white auth for user-agent: nacos-server, only for upgrade from old version. +nacos.core.auth.enable.userAgentAuthWhite=true + +### Since 1.4.1, worked when nacos.core.auth.enabled=true and nacos.core.auth.enable.userAgentAuthWhite=false. +### The two properties is the white list for auth and used by identity the request from other server. +nacos.core.auth.server.identity.key=serverIdentity +nacos.core.auth.server.identity.value=security + +#*************** Istio Related Configurations ***************# +### If turn on the MCP server: +nacos.istio.mcp.server.enabled=false + +#*************** Core Related Configurations ***************# + +### set the WorkerID manually +# nacos.core.snowflake.worker-id= + +### Member-MetaData +# nacos.core.member.meta.site= +# nacos.core.member.meta.adweight= +# nacos.core.member.meta.weight= + +### MemberLookup +### Addressing pattern category, If set, the priority is highest +# nacos.core.member.lookup.type=[file,address-server] +## Set the cluster list with a configuration file or command-line argument +# nacos.member.list=192.168.16.101:8847?raft_port=8807,192.168.16.101?raft_port=8808,192.168.16.101:8849?raft_port=8809 +## for AddressServerMemberLookup +# Maximum number of retries to query the address server upon initialization +# nacos.core.address-server.retry=5 +## Server domain name address of [address-server] mode +# address.server.domain=jmenv.tbsite.net +## Server port of [address-server] mode +# address.server.port=8080 +## Request address of [address-server] mode +# address.server.url=/nacos/serverlist + +#*************** JRaft Related Configurations ***************# + +### Sets the Raft cluster election timeout, default value is 5 second +# nacos.core.protocol.raft.data.election_timeout_ms=5000 +### Sets the amount of time the Raft snapshot will execute periodically, default is 30 minute +# nacos.core.protocol.raft.data.snapshot_interval_secs=30 +### raft internal worker threads +# nacos.core.protocol.raft.data.core_thread_num=8 +### Number of threads required for raft business request processing +# nacos.core.protocol.raft.data.cli_service_thread_num=4 +### raft linear read strategy. Safe linear reads are used by default, that is, the Leader tenure is confirmed by heartbeat +# nacos.core.protocol.raft.data.read_index_type=ReadOnlySafe +### rpc request timeout, default 5 seconds +# nacos.core.protocol.raft.data.rpc_request_timeout_ms=5000 + +#*************** Distro Related Configurations ***************# + +### Distro data sync delay time, when sync task delayed, task will be merged for same data key. Default 1 second. +# nacos.core.protocol.distro.data.sync.delayMs=1000 + +### Distro data sync timeout for one sync data, default 3 seconds. +# nacos.core.protocol.distro.data.sync.timeoutMs=3000 + +### Distro data sync retry delay time when sync data failed or timeout, same behavior with delayMs, default 3 seconds. +# nacos.core.protocol.distro.data.sync.retryDelayMs=3000 + +### Distro data verify interval time, verify synced data whether expired for a interval. Default 5 seconds. +# nacos.core.protocol.distro.data.verify.intervalMs=5000 + +### Distro data verify timeout for one verify, default 3 seconds. +# nacos.core.protocol.distro.data.verify.timeoutMs=3000 + +### Distro data load retry delay when load snapshot data failed, default 30 seconds. +# nacos.core.protocol.distro.data.load.retryDelayMs=30000 diff --git a/parcels/roles/components/nacos/templates/application.properties.j2_1.0 b/parcels/roles/components/nacos/templates/application.properties.j2_1.0 new file mode 100644 index 0000000..27a7656 --- /dev/null +++ b/parcels/roles/components/nacos/templates/application.properties.j2_1.0 @@ -0,0 +1,189 @@ +# +# Copyright 1999-2018 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#*************** Spring Boot Related Configurations ***************# +### Default web context path: +server.servlet.contextPath=/nacos +### Default web server port: +server.port={{ nacos_port }} + +#*************** Network Related Configurations ***************# +### If prefer hostname over ip for Nacos server addresses in cluster.conf: +# nacos.inetutils.prefer-hostname-over-ip=false + +### Specify local server's IP: +# nacos.inetutils.ip-address= + + +#*************** Config Module Related Configurations ***************# +### If use MySQL as datasource: +# spring.datasource.platform=mysql + +### Count of DB: +db.num=1 + +### Connect URL of DB: +db.url.0=jdbc:mysql://{{ gateway_keepalive_host }}:{{ galaxy_mariadb_port }}/{{ mariadb_nacos_database }}?characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useUnicode=true&useSSL=false&serverTimezone=UTC +db.user.0={{ galaxy_mariadb_username }} +db.password.0={{ galaxy_mariadb_pin }} + +### Connection pool configuration: hikariCP +db.pool.config.connectionTimeout=30000 +db.pool.config.validationTimeout=10000 +db.pool.config.maximumPoolSize=20 +db.pool.config.minimumIdle=2 + +#*************** Naming Module Related Configurations ***************# +### Data dispatch task execution period in milliseconds: +# nacos.naming.distro.taskDispatchPeriod=200 + +### Data count of batch sync task: +# nacos.naming.distro.batchSyncKeyCount=1000 + +### Retry delay in milliseconds if sync task failed: +# nacos.naming.distro.syncRetryDelay=5000 + +### If enable data warmup. If set to false, the server would accept request without local data preparation: +# nacos.naming.data.warmup=true + +### If enable the instance auto expiration, kind like of health check of instance: +# nacos.naming.expireInstance=true + +nacos.naming.empty-service.auto-clean=true +nacos.naming.empty-service.clean.initial-delay-ms=50000 +nacos.naming.empty-service.clean.period-time-ms=30000 + + +#*************** CMDB Module Related Configurations ***************# +### The interval to dump external CMDB in seconds: +# nacos.cmdb.dumpTaskInterval=3600 + +### The interval of polling data change event in seconds: +# nacos.cmdb.eventTaskInterval=10 + +### The interval of loading labels in seconds: +# nacos.cmdb.labelTaskInterval=300 + +### If turn on data loading task: +# nacos.cmdb.loadDataAtStart=false + + +#*************** Metrics Related Configurations ***************# +### Metrics for prometheus +management.endpoints.web.exposure.include=* + +### Metrics for elastic search +management.metrics.export.elastic.enabled=false +#management.metrics.export.elastic.host=http://localhost:9200 + +### Metrics for influx +management.metrics.export.influx.enabled=false +#management.metrics.export.influx.db=springboot +#management.metrics.export.influx.uri=http://localhost:8086 +#management.metrics.export.influx.auto-create-db=true +#management.metrics.export.influx.consistency=one +#management.metrics.export.influx.compressed=true + + +#*************** Access Log Related Configurations ***************# +### If turn on the access log: +server.tomcat.accesslog.enabled=true + +### The access log pattern: +server.tomcat.accesslog.pattern=%h %l %u %t "%r" %s %b %D %{User-Agent}i %{Request-Source}i + +### The directory of access log: +server.tomcat.basedir= + + +#*************** Access Control Related Configurations ***************# +### If enable spring security, this option is deprecated in 1.2.0: +#spring.security.enabled=false + +### The ignore urls of auth, is deprecated in 1.2.0: +nacos.security.ignore.urls=/,/error,/**/*.css,/**/*.js,/**/*.html,/**/*.map,/**/*.svg,/**/*.png,/**/*.ico,/console-ui/public/**,/v1/auth/**,/v1/console/health/**,/actuator/**,/v1/console/server/** + +### The auth system to use, currently only 'nacos' is supported: +nacos.core.auth.system.type=nacos + +### If turn on auth system: +nacos.core.auth.enabled=true + +### The token expiration in seconds: +nacos.core.auth.default.token.expire.seconds=18000 + +### The default token: +nacos.core.auth.default.token.secret.key=SecretKey012345678901234567890123456789012345678901234567890123456789 + +### Turn on/off caching of auth information. By turning on this switch, the update of auth information would have a 15 seconds delay. +nacos.core.auth.caching.enabled=true + +### Since 1.4.1, Turn on/off white auth for user-agent: nacos-server, only for upgrade from old version. +nacos.core.auth.enable.userAgentAuthWhite=true + +### Since 1.4.1, worked when nacos.core.auth.enabled=true and nacos.core.auth.enable.userAgentAuthWhite=false. +### The two properties is the white list for auth and used by identity the request from other server. +nacos.core.auth.server.identity.key= +nacos.core.auth.server.identity.value= + +#*************** Istio Related Configurations ***************# +### If turn on the MCP server: +nacos.istio.mcp.server.enabled=false + + + +###*************** Add from 1.3.0 ***************### + + +#*************** Core Related Configurations ***************# + +### set the WorkerID manually +# nacos.core.snowflake.worker-id= + +### Member-MetaData +# nacos.core.member.meta.site= +# nacos.core.member.meta.adweight= +# nacos.core.member.meta.weight= + +### MemberLookup +### Addressing pattern category, If set, the priority is highest +# nacos.core.member.lookup.type=[file,address-server] +## Set the cluster list with a configuration file or command-line argument +# nacos.member.list=192.168.16.101:8847?raft_port=8807,192.168.16.101?raft_port=8808,192.168.16.101:8849?raft_port=8809 +## for AddressServerMemberLookup +# Maximum number of retries to query the address server upon initialization +# nacos.core.address-server.retry=5 +## Server domain name address of [address-server] mode +# address.server.domain=jmenv.tbsite.net +## Server port of [address-server] mode +# address.server.port=8080 +## Request address of [address-server] mode +# address.server.url=/nacos/serverlist + +#*************** JRaft Related Configurations ***************# + +### Sets the Raft cluster election timeout, default value is 5 second +# nacos.core.protocol.raft.data.election_timeout_ms=5000 +### Sets the amount of time the Raft snapshot will execute periodically, default is 30 minute +# nacos.core.protocol.raft.data.snapshot_interval_secs=30 +### raft internal worker threads +# nacos.core.protocol.raft.data.core_thread_num=8 +### Number of threads required for raft business request processing +# nacos.core.protocol.raft.data.cli_service_thread_num=4 +### raft linear read strategy. Safe linear reads are used by default, that is, the Leader tenure is confirmed by heartbeat +# nacos.core.protocol.raft.data.read_index_type=ReadOnlySafe +### rpc request timeout, default 5 seconds +# nacos.core.protocol.raft.data.rpc_request_timeout_ms=5000 diff --git a/parcels/roles/components/nacos/templates/cluster.conf.j2 b/parcels/roles/components/nacos/templates/cluster.conf.j2 new file mode 100644 index 0000000..4127c87 --- /dev/null +++ b/parcels/roles/components/nacos/templates/cluster.conf.j2 @@ -0,0 +1,18 @@ +# +# Copyright 1999-2018 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +#it is ip + diff --git a/parcels/roles/components/nacos/templates/dae-nacos.sh.j2 b/parcels/roles/components/nacos/templates/dae-nacos.sh.j2 new file mode 100755 index 0000000..1a7773a --- /dev/null +++ b/parcels/roles/components/nacos/templates/dae-nacos.sh.j2 @@ -0,0 +1,24 @@ +#!/bin/sh + +#修改BASE_DIR为安装的路径 +#启动命令端口可自行指定 + +PRO_NAME=nacos +BASE_DIR={{ install_path }} +VERSION={{ nacos_version }} +source /etc/profile + + +while true ; do + NUM=`jps | grep -w ${PRO_NAME} | grep -v grep |wc -l` + + if [ "${NUM}" -lt "1" ];then + nohup $BASE_DIR/$VERSION/bin/startup.sh > /dev/null 2>&1 & + OLD_NUM=`cat $BASE_DIR/$VERSION/logs/restart_sum.log` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASE_DIR/$VERSION/logs/restart_sum.log + echo "`date "+%Y-%m-%d %H:%M:%S"` - nacos服务启动/异常重启 - 重启次数 -> $RESTART_NUM" >> $BASE_DIR/$VERSION/restart.log + fi + +sleep 60 +done diff --git a/parcels/roles/components/nacos/templates/keepnacosalive.j2 b/parcels/roles/components/nacos/templates/keepnacosalive.j2 new file mode 100755 index 0000000..6a6f2fa --- /dev/null +++ b/parcels/roles/components/nacos/templates/keepnacosalive.j2 @@ -0,0 +1,42 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 30 70 +# description: keepnacosalive +source /etc/profile +PRO_NAME=keepnacosalive + +case $1 in +start) +keepnum=`ps -ef | grep dae-nacos.sh | grep -v grep | wc -l` + +if [ $keepnum -eq "0" ];then +nohup {{ install_path }}/{{ nacos_version }}/bin/dae-nacos.sh > /dev/null 2>&1 & +fi +;; + +stop) +keeppid=`ps -ef | grep dae-nacos.sh | grep -v grep | awk '{print $2}'` +if [ $keeppid ];then +echo "守护进程PID：$keeppid" +kill -9 $keeppid +sleep 1 +sh {{ install_path }}/{{ nacos_version }}/bin/shutdown.sh +fi +;; + +status) +num=`jps | grep -w nacos | grep -v grep |wc -l` +if [ "$num" -eq "1" ];then +echo "nacos进程已启动" +else +echo "nacos进程未启动" +fi +;; + +* ) +echo "use keepnacosalive [start|stop|status]" +;; +esac + diff --git a/parcels/roles/components/nacos/templates/nacos-logback.xml.j2 b/parcels/roles/components/nacos/templates/nacos-logback.xml.j2 new file mode 100644 index 0000000..289a590 --- /dev/null +++ b/parcels/roles/components/nacos/templates/nacos-logback.xml.j2 @@ -0,0 +1,642 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Copyright 1999-2018 Alibaba Group Holding Ltd. + ~ + ~ Licensed under the Apache License, Version 2.0 (the "License"); + ~ you may not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<configuration scan="true" scanPeriod="10 seconds"> + + <springProperty scope="context" name="logPath" source="nacos.logs.path" defaultValue="${nacos.home}/logs"/> + <property name="LOG_HOME" value="${logPath}"/> + + <appender name="cmdb-main" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${nacos.home}/logs/cmdb-main.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${nacos.home}/logs/cmdb-main.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>2GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>7GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="naming-server" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/naming-server.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/naming-server.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>1GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>7GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="async-naming-server" class="ch.qos.logback.classic.AsyncAppender"> + <discardingThreshold>0</discardingThreshold> + <queueSize>1024</queueSize> + <neverBlock>true</neverBlock> + <appender-ref ref="naming-server"/> + </appender> + + <appender name="naming-raft" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/naming-raft.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/naming-raft.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>1GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="async-naming-raft" class="ch.qos.logback.classic.AsyncAppender"> + <discardingThreshold>0</discardingThreshold> + <queueSize>1024</queueSize> + <neverBlock>true</neverBlock> + <appender-ref ref="naming-raft"/> + </appender> + + + <appender name="naming-distro" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/naming-distro.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/naming-distro.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>1GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="async-naming-distro" class="ch.qos.logback.classic.AsyncAppender"> + <discardingThreshold>0</discardingThreshold> + <queueSize>1024</queueSize> + <neverBlock>true</neverBlock> + <appender-ref ref="naming-distro"/> + </appender> + + <appender name="naming-event" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/naming-event.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/naming-event.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>1GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="async-naming-event" class="ch.qos.logback.classic.AsyncAppender"> + <discardingThreshold>0</discardingThreshold> + <queueSize>1024</queueSize> + <neverBlock>true</neverBlock> + <appender-ref ref="naming-event"/> + </appender> + + <appender name="naming-push" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/naming-push.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/naming-push.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>1GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + <appender name="naming-rt" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/naming-rt.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/naming-rt.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>1GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%msg%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="naming-performance" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/naming-performance.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/naming-performance.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>1GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <!--config module logback config--> + <appender name="dumpFile" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/config-dump.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/config-dump.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>2GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>7GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + <appender name="pullFile" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/config-pull.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/config-pull.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>20MB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>128MB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + <appender name="fatalFile" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/config-fatal.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/config-fatal.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>20MB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>128MB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + <appender name="memoryFile" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/config-memory.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/config-memory.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>20MB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>128MB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + <appender name="pullCheckFile" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/config-pull-check.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/config-pull-check.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>1GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%msg%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="clientLog" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/config-client-request.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/config-client-request.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>2GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>7GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date|%msg%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="traceLog" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/config-trace.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/config-trace.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>2GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>7GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date|%msg%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="notifyLog" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/config-notify.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/config-notify.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>1GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>3GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="startLog" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/config-server.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/config-server.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>50MB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>512MB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="rootFile" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/nacos.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/nacos.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>50MB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>512MB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="nacos-address" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/nacos-address.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/nacos-address.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>2GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>7GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="istio-main" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/istio-main.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/istio-main.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>2GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>7GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="core-auth" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/core-auth.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/core-auth.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>2GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>7GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="protocol-raft" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/protocol-raft.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/protocol-raft.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>2GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>7GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="protocol-distro" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/protocol-distro.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/protocol-distro.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>2GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>7GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="nacos-cluster" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/nacos-cluster.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/nacos-cluster.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>2GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>7GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <appender name="alipay-jraft" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_HOME}/alipay-jraft.log</file> + <append>true</append> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern>${LOG_HOME}/alipay-jraft.log.%d{yyyy-MM-dd}.%i</fileNamePattern> + <maxFileSize>2GB</maxFileSize> + <maxHistory>7</maxHistory> + <totalSizeCap>7GB</totalSizeCap> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + <encoder> + <Pattern>%date %level %msg%n%n</Pattern> + <charset>UTF-8</charset> + </encoder> + </appender> + + <logger name="com.alibaba.nacos.address.main" additivity="false"> + <level value="INFO"/> + <appender-ref ref="nacos-address"/> + </logger> + + <logger name="com.alibaba.nacos.cmdb.main" additivity="false"> + <level value="INFO"/> + <appender-ref ref="cmdb-main"/> + </logger> + + <logger name="com.alibaba.nacos.naming.main" additivity="false"> + <level value="INFO"/> + <appender-ref ref="async-naming-server"/> + </logger> + <logger name="com.alibaba.nacos.naming.raft" additivity="false"> + <level value="INFO"/> + <appender-ref ref="async-naming-raft"/> + </logger> + <logger name="com.alibaba.nacos.naming.distro" additivity="false"> + <level value="INFO"/> + <appender-ref ref="async-naming-distro"/> + </logger> + <logger name="com.alibaba.nacos.naming.event" additivity="false"> + <level value="INFO"/> + <appender-ref ref="async-naming-event"/> + </logger> + <logger name="com.alibaba.nacos.naming.push" additivity="false"> + <level value="INFO"/> + <appender-ref ref="naming-push"/> + </logger> + <logger name="com.alibaba.nacos.naming.rt" additivity="false"> + <level value="INFO"/> + <appender-ref ref="naming-rt"/> + </logger> + <logger name="com.alibaba.nacos.naming.performance" additivity="false"> + <level value="INFO"/> + <appender-ref ref="naming-performance"/> + </logger> + + <logger name="com.alibaba.nacos.config.dumpLog" additivity="false"> + <level value="INFO"/> + <appender-ref ref="dumpFile"/> + </logger> + <logger name="com.alibaba.nacos.config.pullLog" additivity="false"> + <level value="INFO"/> + <appender-ref ref="pullFile"/> + </logger> + <logger name="com.alibaba.nacos.config.pullCheckLog" additivity="false"> + <level value="INFO"/> + <appender-ref ref="pullCheckFile"/> + </logger> + <logger name="com.alibaba.nacos.config.fatal" additivity="false"> + <level value="INFO"/> + <appender-ref ref="fatalFile"/> + </logger> + <logger name="com.alibaba.nacos.config.monitorLog" additivity="false"> + <level value="INFO"/> + <appender-ref ref="memoryFile"/> + </logger> + + <logger name="com.alibaba.nacos.config.clientLog" additivity="false"> + <level value="info"/> + <appender-ref ref="clientLog"/> + </logger> + + <logger name="com.alibaba.nacos.config.notifyLog" additivity="false"> + <level value="INFO"/> + <appender-ref ref="notifyLog"/> + </logger> + + <logger name="com.alibaba.nacos.config.traceLog" additivity="false"> + <level value="info"/> + <appender-ref ref="traceLog"/> + </logger> + + <logger name="com.alibaba.nacos.config.startLog" additivity="false"> + <level value="INFO"/> + <appender-ref ref="startLog"/> + </logger> + + <logger name="com.alibaba.nacos.istio.main" additivity="false"> + <level value="DEBUG"/> + <appender-ref ref="istio-main"/> + </logger> + + <logger name="com.alibaba.nacos.core.auth" additivity="false"> + <level value="DEBUG"/> + <appender-ref ref="core-auth"/> + </logger> + + <logger name="com.alibaba.nacos.core.protocol.raft" additivity="false"> + <level value="INFO"/> + <appender-ref ref="protocol-raft"/> + </logger> + + <logger name="com.alipay.sofa.jraft" additivity="false"> + <level value="INFO"/> + <appender-ref ref="alipay-jraft"/> + </logger> + + <logger name="com.alibaba.nacos.core.protocol.distro" additivity="false"> + <level value="INFO"/> + <appender-ref ref="protocol-distro"/> + </logger> + + <logger name="com.alibaba.nacos.core.cluster" additivity="false"> + <level value="INFO"/> + <appender-ref ref="nacos-cluster"/> + </logger> + + <springProfile name="standalone"> + <logger name="org.springframework"> + <appender-ref ref="CONSOLE"/> + <level value="INFO"/> + </logger> + + <logger name="org.apache.catalina.startup.DigesterFactory"> + <appender-ref ref="CONSOLE"/> + <level value="INFO"/> + </logger> + + <logger name="org.apache.catalina.util.LifecycleBase"> + <appender-ref ref="CONSOLE"/> + <level value="ERROR"/> + </logger> + + <logger name="org.apache.coyote.http11.Http11NioProtocol"> + <appender-ref ref="CONSOLE"/> + <level value="WARN"/> + </logger> + + <logger name="org.apache.tomcat.util.net.NioSelectorPool"> + <appender-ref ref="CONSOLE"/> + <level value="WARN"/> + </logger> + </springProfile> + + <logger name="com.alibaba.nacos.core.listener.StartingApplicationListener"> + <appender-ref ref="CONSOLE"/> + <level value="INFO"/> + </logger> + + <logger name="com.alibaba.nacos.common.notify.NotifyCenter"> + <appender-ref ref="CONSOLE"/> + <level value="INFO"/> + </logger> + + <logger name="com.alibaba.nacos.sys.file.WatchFileCenter"> + <appender-ref ref="CONSOLE"/> + <level value="INFO"/> + </logger> + + <logger name="com.alibaba.nacos.common.executor.ThreadPoolManager"> + <appender-ref ref="CONSOLE"/> + <level value="INFO"/> + </logger> + + <root> + <level value="INFO"/> + <appender-ref ref="rootFile"/> + </root> +</configuration> + diff --git a/parcels/roles/components/nacos/templates/nacos-mysql.sql.j2 b/parcels/roles/components/nacos/templates/nacos-mysql.sql.j2 new file mode 100644 index 0000000..273ecc5 --- /dev/null +++ b/parcels/roles/components/nacos/templates/nacos-mysql.sql.j2 @@ -0,0 +1,226 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/******************************************/ +/* 数据库全名 = nacos_config */ +/* 表名称 = config_info */ +/******************************************/ +CREATE TABLE `config_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id', + `data_id` varchar(255) NOT NULL COMMENT 'data_id', + `group_id` varchar(255) DEFAULT NULL, + `content` longtext NOT NULL COMMENT 'content', + `md5` varchar(32) DEFAULT NULL COMMENT 'md5', + `gmt_create` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `gmt_modified` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间', + `src_user` text COMMENT 'source user', + `src_ip` varchar(50) DEFAULT NULL COMMENT 'source ip', + `app_name` varchar(128) DEFAULT NULL, + `tenant_id` varchar(128) DEFAULT '' COMMENT '租户字段', + `c_desc` varchar(256) DEFAULT NULL, + `c_use` varchar(64) DEFAULT NULL, + `effect` varchar(64) DEFAULT NULL, + `type` varchar(64) DEFAULT NULL, + `c_schema` text, + PRIMARY KEY (`id`), + UNIQUE KEY `uk_configinfo_datagrouptenant` (`data_id`,`group_id`,`tenant_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='config_info'; + +/******************************************/ +/* 数据库全名 = nacos_config */ +/* 表名称 = config_info_aggr */ +/******************************************/ +CREATE TABLE `config_info_aggr` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id', + `data_id` varchar(255) NOT NULL COMMENT 'data_id', + `group_id` varchar(255) NOT NULL COMMENT 'group_id', + `datum_id` varchar(255) NOT NULL COMMENT 'datum_id', + `content` longtext NOT NULL COMMENT '内容', + `gmt_modified` datetime NOT NULL COMMENT '修改时间', + `app_name` varchar(128) DEFAULT NULL, + `tenant_id` varchar(128) DEFAULT '' COMMENT '租户字段', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_configinfoaggr_datagrouptenantdatum` (`data_id`,`group_id`,`tenant_id`,`datum_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='增加租户字段'; + + +/******************************************/ +/* 数据库全名 = nacos_config */ +/* 表名称 = config_info_beta */ +/******************************************/ +CREATE TABLE `config_info_beta` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id', + `data_id` varchar(255) NOT NULL COMMENT 'data_id', + `group_id` varchar(128) NOT NULL COMMENT 'group_id', + `app_name` varchar(128) DEFAULT NULL COMMENT 'app_name', + `content` longtext NOT NULL COMMENT 'content', + `beta_ips` varchar(1024) DEFAULT NULL COMMENT 'betaIps', + `md5` varchar(32) DEFAULT NULL COMMENT 'md5', + `gmt_create` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `gmt_modified` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间', + `src_user` text COMMENT 'source user', + `src_ip` varchar(50) DEFAULT NULL COMMENT 'source ip', + `tenant_id` varchar(128) DEFAULT '' COMMENT '租户字段', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_configinfobeta_datagrouptenant` (`data_id`,`group_id`,`tenant_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='config_info_beta'; + +/******************************************/ +/* 数据库全名 = nacos_config */ +/* 表名称 = config_info_tag */ +/******************************************/ +CREATE TABLE `config_info_tag` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id', + `data_id` varchar(255) NOT NULL COMMENT 'data_id', + `group_id` varchar(128) NOT NULL COMMENT 'group_id', + `tenant_id` varchar(128) DEFAULT '' COMMENT 'tenant_id', + `tag_id` varchar(128) NOT NULL COMMENT 'tag_id', + `app_name` varchar(128) DEFAULT NULL COMMENT 'app_name', + `content` longtext NOT NULL COMMENT 'content', + `md5` varchar(32) DEFAULT NULL COMMENT 'md5', + `gmt_create` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `gmt_modified` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间', + `src_user` text COMMENT 'source user', + `src_ip` varchar(50) DEFAULT NULL COMMENT 'source ip', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_configinfotag_datagrouptenanttag` (`data_id`,`group_id`,`tenant_id`,`tag_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='config_info_tag'; + +/******************************************/ +/* 数据库全名 = nacos_config */ +/* 表名称 = config_tags_relation */ +/******************************************/ +CREATE TABLE `config_tags_relation` ( + `id` bigint(20) NOT NULL COMMENT 'id', + `tag_name` varchar(128) NOT NULL COMMENT 'tag_name', + `tag_type` varchar(64) DEFAULT NULL COMMENT 'tag_type', + `data_id` varchar(255) NOT NULL COMMENT 'data_id', + `group_id` varchar(128) NOT NULL COMMENT 'group_id', + `tenant_id` varchar(128) DEFAULT '' COMMENT 'tenant_id', + `nid` bigint(20) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`nid`), + UNIQUE KEY `uk_configtagrelation_configidtag` (`id`,`tag_name`,`tag_type`), + KEY `idx_tenant_id` (`tenant_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='config_tag_relation'; + +/******************************************/ +/* 数据库全名 = nacos_config */ +/* 表名称 = group_capacity */ +/******************************************/ +CREATE TABLE `group_capacity` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `group_id` varchar(128) NOT NULL DEFAULT '' COMMENT 'Group ID，空字符表示整个集群', + `quota` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '配额，0表示使用默认值', + `usage` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '使用量', + `max_size` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '单个配置大小上限，单位为字节，0表示使用默认值', + `max_aggr_count` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '聚合子配置最大个数，，0表示使用默认值', + `max_aggr_size` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '单个聚合数据的子配置大小上限，单位为字节，0表示使用默认值', + `max_history_count` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '最大变更历史数量', + `gmt_create` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `gmt_modified` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_group_id` (`group_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='集群、各Group容量信息表'; + +/******************************************/ +/* 数据库全名 = nacos_config */ +/* 表名称 = his_config_info */ +/******************************************/ +CREATE TABLE `his_config_info` ( + `id` bigint(64) unsigned NOT NULL, + `nid` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `data_id` varchar(255) NOT NULL, + `group_id` varchar(128) NOT NULL, + `app_name` varchar(128) DEFAULT NULL COMMENT 'app_name', + `content` longtext NOT NULL, + `md5` varchar(32) DEFAULT NULL, + `gmt_create` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `gmt_modified` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `src_user` text, + `src_ip` varchar(50) DEFAULT NULL, + `op_type` char(10) DEFAULT NULL, + `tenant_id` varchar(128) DEFAULT '' COMMENT '租户字段', + PRIMARY KEY (`nid`), + KEY `idx_gmt_create` (`gmt_create`), + KEY `idx_gmt_modified` (`gmt_modified`), + KEY `idx_did` (`data_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='多租户改造'; + + +/******************************************/ +/* 数据库全名 = nacos_config */ +/* 表名称 = tenant_capacity */ +/******************************************/ +CREATE TABLE `tenant_capacity` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `tenant_id` varchar(128) NOT NULL DEFAULT '' COMMENT 'Tenant ID', + `quota` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '配额，0表示使用默认值', + `usage` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '使用量', + `max_size` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '单个配置大小上限，单位为字节，0表示使用默认值', + `max_aggr_count` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '聚合子配置最大个数', + `max_aggr_size` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '单个聚合数据的子配置大小上限，单位为字节，0表示使用默认值', + `max_history_count` int(10) unsigned NOT NULL DEFAULT '0' COMMENT '最大变更历史数量', + `gmt_create` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `gmt_modified` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_tenant_id` (`tenant_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='租户容量信息表'; + + +CREATE TABLE `tenant_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id', + `kp` varchar(128) NOT NULL COMMENT 'kp', + `tenant_id` varchar(128) default '' COMMENT 'tenant_id', + `tenant_name` varchar(128) default '' COMMENT 'tenant_name', + `tenant_desc` varchar(256) DEFAULT NULL COMMENT 'tenant_desc', + `create_source` varchar(32) DEFAULT NULL COMMENT 'create_source', + `gmt_create` bigint(20) NOT NULL COMMENT '创建时间', + `gmt_modified` bigint(20) NOT NULL COMMENT '修改时间', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_tenant_info_kptenantid` (`kp`,`tenant_id`), + KEY `idx_tenant_id` (`tenant_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='tenant_info'; + +CREATE TABLE `users` ( + `username` varchar(50) NOT NULL PRIMARY KEY, + `password` varchar(500) NOT NULL, + `enabled` boolean NOT NULL +); + +CREATE TABLE `roles` ( + `username` varchar(50) NOT NULL, + `role` varchar(50) NOT NULL, + UNIQUE INDEX `idx_user_role` (`username` ASC, `role` ASC) USING BTREE +); + +CREATE TABLE `permissions` ( + `role` varchar(50) NOT NULL, + `resource` varchar(255) NOT NULL, + `action` varchar(8) NOT NULL, + UNIQUE INDEX `uk_role_permission` (`role`,`resource`,`action`) USING BTREE +); + +INSERT INTO users (username, password, enabled) VALUES ('nacos', '$2a$10$EuWPZHzz32dJN7jexM34MOeYirDdFAZm2kuWj7VEOJhhZkDrxfvUu', TRUE); + +INSERT INTO roles (username, role) VALUES ('nacos', 'ROLE_ADMIN'); + + +-- ---------------------------- +-- Records of tenant_info +-- ---------------------------- +INSERT INTO `tenant_info` VALUES (null, '1', 'dev', 'dev', '开发环境', 'nacos', 1616642477210, 1616642477210); +INSERT INTO `tenant_info` VALUES (null, '1', 'test', 'test', '测试环境', 'nacos', 1616642477210, 1616642477210); +INSERT INTO `tenant_info` VALUES (null, '1', 'prod', 'prod', '生产环境', 'nacos', 1616642477210, 1616642477210); diff --git a/parcels/roles/components/nacos/templates/nacos-mysql.sql.j2_20210727 b/parcels/roles/components/nacos/templates/nacos-mysql.sql.j2_20210727 new file mode 100644 index 0000000..9115b12 --- /dev/null +++ b/parcels/roles/components/nacos/templates/nacos-mysql.sql.j2_20210727 @@ -0,0 +1,303 @@ +/* + Navicat Premium Data Transfer + + Source Server : 192.168.40.203 + Source Server Type : MariaDB + Source Server Version : 100226 + Source Host : 192.168.40.203:3306 + Source Schema : nacos + + Target Server Type : MariaDB + Target Server Version : 100226 + File Encoding : 65001 + + Date: 03/06/2021 16:52:01 +*/ + +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for config_info +-- ---------------------------- +DROP TABLE IF EXISTS `config_info`; +CREATE TABLE `config_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id', + `data_id` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'data_id', + `group_id` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL, + `content` longtext CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'content', + `md5` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'md5', + `gmt_create` datetime(0) NOT NULL DEFAULT current_timestamp() COMMENT '创建时间', + `gmt_modified` datetime(0) NOT NULL DEFAULT current_timestamp() COMMENT '修改时间', + `src_user` text CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'source user', + `src_ip` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'source ip', + `app_name` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL, + `tenant_id` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT '' COMMENT '租户字段', + `c_desc` varchar(256) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL, + `c_use` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL, + `effect` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL, + `type` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL, + `c_schema` text CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE, + UNIQUE INDEX `uk_configinfo_datagrouptenant`(`data_id`, `group_id`, `tenant_id`) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 75 CHARACTER SET = utf8 COLLATE = utf8_bin COMMENT = 'config_info' ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Records of config_info +-- ---------------------------- +INSERT INTO `config_info` VALUES (3, 'ck-queries-template.sql', 'Galaxy', '--Q01.Count(1)\nselect count(1) from connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end)\n--Q02.All Fields Query (default)\nSELECT * FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) LIMIT 30\n--Q03.All Fields Query order by Time desc\nSELECT * FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30\n--Q04.All Fields Query order by Time asc\nSELECT * FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time asc LIMIT 30\n--Q05.All Fields Query by Filter\nSELECT * FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @common_filter ORDER BY common_recv_time DESC LIMIT 30\n--Q06.Default Fields Query by Filter\nSELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @common_filter ORDER BY common_recv_time DESC LIMIT 30\n--Q07.All Fields Query (sub query by time)\nSELECT * FROM connection_record_log AS connection_record_log WHERE toDateTime(common_recv_time) IN ( SELECT toDateTime(common_recv_time) FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30\n--Q08.All Fields Query (sub query by log id)\nSELECT * FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( SELECT common_log_id FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30\n--Q09.Default Field Query (sub query by time)\nSELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE toDateTime(common_recv_time) IN ( SELECT toDateTime(common_recv_time) FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30\n--Q10.Default Field Query (sub query by log id)\nSELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( select common_log_id FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end)) ORDER BY common_recv_time DESC LIMIT 30\n--Q11.Default Field Query by Server IP (sub query by log id with Index Table)\nSELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( Select common_log_id FROM connection_record_log_common_server_ip AS connection_record_log_common_server_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30\n--Q12.Default Field Query by Client IP (sub query by log id with Index Table)\nSELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( Select common_log_id FROM connection_record_log_common_client_ip AS connection_record_log_common_client_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30\n--Q13.Default Field Query by Domain (sub query by log id with Index Table)\nSELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( Select common_log_id FROM connection_record_log_http_domain AS connection_record_log_http_domain WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30\n--Q14.Default Field Query by Subscriber ID (sub query by log id with Index Table)\nSELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( Select common_log_id FROM connection_record_log_common_subscriber_id AS connection_record_log_common_subscriber_id WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30\n--Q15.All Fields Query by Client IP (sub query by log id with index Table)\nSELECT * FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( SELECT common_log_id FROM connection_record_log_common_client_ip AS connection_record_log_common_client_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30\n--Q16.All Fields Query by Server IP(sub query by log id with index Table)\nSELECT * FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( SELECT common_log_id FROM connection_record_log_common_server_ip AS connection_record_log_common_server_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY common_recv_time LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30\n--Q17.All Fields Query by Domain(sub query by log id with index Table)\nSELECT * FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( SELECT common_log_id FROM connection_record_log_http_domain AS connection_record_log_http_domain WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY common_recv_time LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30\n--Q18.All Fields Query by Subscriber ID(sub query by log id with index Table)\nSELECT * FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( SELECT common_log_id FROM connection_record_log_common_subscriber_id AS connection_record_log_common_subscriber_id WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY common_recv_time LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30\n--Q19.Session Logs Sent to Database Trend(Time Grain 5 minute)\nSELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS \"Receive Time\", count(common_log_id) AS \"logs\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ) ) GROUP BY \"Receive Time\" LIMIT 10000\n--Q20.Traffic Bandwidth Trend(Time Grain 30 second)\nSELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 30 SECOND)))) AS stat_time, sum(common_c2s_byte_num) AS bytes_sent, sum(common_s2c_byte_num) AS bytes_received, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets, sum(common_sessions) AS sessions FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY stat_time ORDER BY stat_time ASC LIMIT 10000\n--Q21.Log Tend by Type (Time Grain 5 minute)\nSELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE)))) AS stat_time, common_schema_type AS type, sum(common_sessions) AS sessions, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) GROUP BY stat_time, common_schema_type ORDER BY stat_time ASC LIMIT 10000\n--Q22.Traffic Metrics Analytic\nSELECT round(sum(common_s2c_byte_num) * 8 / 300,2) AS trafficInBits, round(sum(common_c2s_byte_num) * 8 / 300,2) AS trafficOutBits, round(sum(common_s2c_byte_num + common_c2s_byte_num) * 8 / 300,2) AS trafficTotalBits, round(sum(common_s2c_pkt_num) / 300,2) AS trafficInPackets, round(sum(common_c2s_pkt_num) / 300,2) AS trafficOutPackets, round(sum(common_s2c_pkt_num + common_c2s_pkt_num) / 300,2) AS trafficTotalPackets, round(sum(common_sessions) / 300,2) AS sessions FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end)\n--Q23.Traffic Endpoints Metrics Trend(Time Grain 5 minute)\nSELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS \"Receive Time\", uniq(common_internal_ip) AS \"Unique Internal IP\", uniq(common_external_ip) AS \"Unique External IP\", uniq(common_subscriber_id) AS \"Unique Subscriber ID\", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS \"Bytes\", sum(coalesce(common_sessions, 0)) AS \"Sessions\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY \"Receive Time\" LIMIT 10000\n--Q24.Endpoint Unique Num by L4 Protocol\nSELECT \'all\' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) UNION ALL SELECT \'tcp\' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_l4_protocol IN ( \'IPv4_TCP\', \'IPv6_TCP\' ) UNION ALL SELECT \'UDP\' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_l4_protocol IN ( \'IPv4_UDP\', \'IPv6_UDP\' )\n--Q25.One-sided Connection Trend(Time Grain 5 minute)\nSELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE)))) AS stat_time, (CASE WHEN common_stream_dir = 1 THEN \'c2s\' WHEN common_stream_dir = 2 THEN \'s2c\' WHEN common_stream_dir = 3 THEN \'double\' ELSE \'None\' END) AS type, sum(common_sessions) AS sessions FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY stat_time, common_stream_dir ORDER BY stat_time ASC LIMIT 10000\n--Q26. Estimated One-sided Sessions with Bandwidth\nSELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS \"Receive Time\", sum(common_sessions) AS \"sessions\", sum(if(common_stream_dir <> 3, common_sessions, 0)) AS \"one_side_sessions\", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS \"Bytes\", round(one_side_sessions / sessions, 2) AS one_side_percent FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY \"Receive Time\" LIMIT 10000\n--Q27.Estimated TCP Sequence Gap Loss\nSELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS \"Receive Time\", sum(common_c2s_byte_num + common_s2c_byte_num) AS \"bytes\", sum(common_c2s_tcp_lostlen + common_s2c_tcp_lostlen) AS \"gap_loss_bytes\", round(gap_loss_bytes / bytes, 2) AS gap_loss_percent FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_l4_protocol IN ( \'IPv4_TCP\', \'IPv6_TCP\' ) ) GROUP BY \"Receive Time\" LIMIT 10000\n--Q28.Top30 Server IP by Bytes\nSELECT \"server_ip\" AS \"server_ip\" , SUM(coalesce(\"bytes\",0)) AS \"bytes\" , SUM(coalesce(\"bytes_sent\",0)) AS \"Sent\" , SUM(coalesce(\"bytes_received\",0)) AS \"Received\" , SUM(coalesce(\"sessions\",0)) AS \"sessions\" FROM ( SELECT SUM(coalesce(common_c2s_byte_num,0)) AS \"bytes_sent\" , SUM(coalesce(common_s2c_byte_num,0)) AS \"bytes_received\" , SUM(common_c2s_byte_num+common_s2c_byte_num) AS \"bytes\" , SUM(coalesce(common_sessions,0)) AS \"sessions\" , common_server_ip AS \"server_ip\" FROM connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( common_server_ip) ) GROUP BY \"server_ip\" ORDER BY \"bytes\" desc ) GROUP BY \"server_ip\" ORDER BY \"bytes\" desc LIMIT 30\n--Q29.Top30 Client IP by Sessions\nSELECT common_client_ip , COUNT(*) AS sessions FROM connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY common_client_ip ORDER BY sessions desc LIMIT 0,30\n--Q30.Top30 TCP Server Ports by Sessions\nSELECT \"Server Port\" AS \"Server Port\", sum(coalesce(\"Sessions\", 0)) AS \"Sessions\" FROM (SELECT common_server_port AS \"Server Port\", sum(coalesce(common_sessions, 0)) AS \"Sessions\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_l4_protocol IN ( \'IPv4_TCP\', \'IPv6_TCP\' ) ) GROUP BY \"Server Port\" LIMIT 1048576) GROUP BY \"Server Port\" ORDER BY \"Sessions\" DESC LIMIT 30\n--Q31.Top30 Domian by Bytes\nSELECT \"domain\" AS \"Website Domain\" , SUM(coalesce(\"bytes\",0)) AS \"Throughput\" FROM ( SELECT SUM(coalesce(common_c2s_byte_num,0)) AS \"bytes_sent\" , SUM(coalesce(common_s2c_byte_num,0)) AS \"bytes_received\" , SUM(coalesce(common_c2s_byte_num+common_s2c_byte_num,0)) AS \"bytes\" , http_domain AS \"domain\" FROM connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( http_domain) ) GROUP BY \"domain\" ORDER BY \"bytes\" desc ) GROUP BY \"domain\" ORDER BY \"Throughput\" desc LIMIT 30\n--Q32.Top30 Endpoint Devices by Bandwidth\nSELECT \"device_id\" AS \"device_id\", sum(coalesce(\"bytes\", 0)) AS \"bytes\", sum(coalesce(\"bytes_sent\", 0)) AS \"Sent\", sum(coalesce(\"bytes_received\", 0)) AS \"Received\" FROM (SELECT sum(coalesce(common_c2s_byte_num, 0)) AS \"bytes_sent\", sum(coalesce(common_s2c_byte_num, 0)) AS \"bytes_received\", sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, common_device_id AS \"device_id\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY \"device_id\" ORDER BY \"bytes\" DESC LIMIT 1048576) GROUP BY \"device_id\" ORDER BY \"bytes\" DESC LIMIT 30\n--Q33.Top30 Domain by Unique Client IP\nSELECT \"Http.Domain\" AS \"Http.Domain\", sum(coalesce(\"Client IP\", 0)) AS \"Client IP\" FROM (SELECT http_domain AS \"Http.Domain\", uniq(common_client_ip) AS \"Client IP\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY \"Http.Domain\" ORDER BY \"Client IP\" DESC LIMIT 1048576) GROUP BY \"Http.Domain\" ORDER BY \"Client IP\" DESC LIMIT 30\n--Q34.Top100 Most Time Consuming Domains\nSELECT \"Domain\" AS \"Domain\", avg(coalesce(\"Avg Establish Latency(ms)\", 0)) AS \"Avg Establish Latency(ms)\" FROM (SELECT http_domain AS \"Domain\", avg(coalesce(common_establish_latency_ms, 0)) AS \"Avg Establish Latency(ms)\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY \"Domain\" LIMIT 1048576) GROUP BY \"Domain\" ORDER BY \"Avg Establish Latency(ms)\" DESC LIMIT 100\n--Q35.Top30 Sources by Sessions\nSELECT \"source\" AS \"source\", sum(coalesce(\"sessions\", 0)) AS \"sessions\" FROM (SELECT coalesce(nullif(common_subscriber_id, \'\'), nullif(common_client_ip, \'\')) AS \"source\", sum(coalesce(common_sessions, 0)) AS \"sessions\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY \"source\" ORDER BY \"sessions\" DESC LIMIT 1048576) GROUP BY \"source\" ORDER BY \"sessions\" DESC LIMIT 30\n--Q36.Top30 Destinations by Sessions\nSELECT \"destination\" AS \"destination\", sum(coalesce(\"sessions\", 0)) AS \"sessions\" FROM (SELECT coalesce(nullif(http_domain, \'\'), nullif(common_server_ip, \'\')) AS \"destination\", sum(coalesce(common_sessions, 0)) AS \"sessions\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY \"destination\" ORDER BY \"sessions\" DESC LIMIT 1048576) GROUP BY \"destination\" ORDER BY \"sessions\" DESC LIMIT 30\n--Q37.Top30 Destination Regions by Bandwidth\nSELECT \"server_location\" AS \"server_location\", sum(coalesce(\"bytes\", 0)) AS \"bytes\", sum(coalesce(\"bytes_sent\", 0)) AS \"Sent\", sum(coalesce(\"bytes_received\", 0)) AS \"Received\" FROM (SELECT arrayElement(splitByString(\',\', common_server_location), length(splitByString(\',\', common_server_location))) AS \"server_location\", sum(coalesce(common_c2s_byte_num, 0)) AS \"bytes_sent\", sum(coalesce(common_s2c_byte_num, 0)) AS \"bytes_received\", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS \"bytes\", sum(coalesce(common_sessions, 0)) AS \"sessions\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY \"server_location\" ORDER BY \"bytes\" DESC LIMIT 1048576) GROUP BY \"server_location\" ORDER BY \"bytes\" DESC LIMIT 30\n--Q38.Top30 URLS by Sessions\nSELECT \"Http URL\" AS \"Http URL\", sum(coalesce(\"Sessions\", 0)) AS \"Sessions\" FROM (SELECT http_url AS \"Http URL\", sum(coalesce(common_sessions, 0)) AS \"Sessions\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY \"Http URL\" LIMIT 1048576) GROUP BY \"Http URL\" ORDER BY \"Sessions\" DESC LIMIT 30\n--Q39.Top30 Destination Transmission APP by Bandwidth\nSELECT \"server_ip\" AS \"server_ip\", groupUniqArray(coalesce(\"trans_app\", 0)) AS \"trans_app\", sum(coalesce(\"bytes\", 0)) AS \"bytes\", sum(coalesce(\"bytes_sent\", 0)) AS \"Sent\", sum(coalesce(\"bytes_received\", 0)) AS \"Received\" FROM (SELECT sum(coalesce(common_c2s_byte_num, 0)) AS \"bytes_sent\", sum(coalesce(common_s2c_byte_num, 0)) AS \"bytes_received\", sum(common_c2s_byte_num + common_s2c_byte_num) AS \"bytes\", groupUniqArray(concat(common_l4_protocol, \'/\', toString(common_server_port))) AS \"trans_app\", common_server_ip AS \"server_ip\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(common_server_ip) ) GROUP BY \"server_ip\" ORDER BY \"bytes\" DESC LIMIT 1048576) GROUP BY \"server_ip\" ORDER BY \"bytes\" DESC LIMIT 30\n--Q40.Browsing Users by Website domains and Sessions\nSELECT \"Subscriber ID\" AS \"Subscriber ID\", \"Http.Domain\" AS \"Http.Domain\", sum(coalesce(\"sessions\", 0)) AS \"sessions\" FROM (SELECT http_domain AS \"Http.Domain\", common_subscriber_id AS \"Subscriber ID\", sum(coalesce(common_sessions, 0)) AS \"sessions\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) AND notEmpty(common_subscriber_id) ) GROUP BY \"Http.Domain\", \"Subscriber ID\" ORDER BY \"sessions\" DESC LIMIT 1048576) GROUP BY \"Subscriber ID\", \"Http.Domain\" ORDER BY \"sessions\" DESC LIMIT 10000\n--Q41.Top Domain and Server IP by Bytes Sent\nSELECT \"Http.Domain\" AS \"Http.Domain\" , \"Server IP\" AS \"Server IP\" , SUM(coalesce(\"Bytes Sent\",0)) AS \"Bytes Sent\" FROM ( SELECT common_server_ip AS \"Server IP\" , http_domain AS \"Http.Domain\" , SUM(coalesce(common_c2s_byte_num+common_s2c_byte_num,0)) AS \"Bytes\" , SUM(coalesce(common_c2s_byte_num,0)) AS \"Bytes Sent\" , SUM(coalesce(common_s2c_byte_num,0)) AS \"Bytes Received\" FROM connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( http_domain) ) GROUP BY \"Server IP\" , \"Http.Domain\" ORDER BY \"Bytes\" desc LIMIT 1048576 ) GROUP BY \"Http.Domain\" , \"Server IP\" ORDER BY \"Bytes Sent\" desc LIMIT 10000\n--Q42.Top30 Website Domains by Client IP and Sessions\nSELECT \"Http.Domain\" AS \"Http.Domain\", \"Client IP\" AS \"Client IP\", sum(coalesce(\"sessions\", 0)) AS \"sessions\" FROM (SELECT common_client_ip AS \"Client IP\", http_domain AS \"Http.Domain\", sum(coalesce(common_sessions, 0)) AS \"sessions\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY \"Client IP\", \"Http.Domain\" ORDER BY \"sessions\" DESC LIMIT 1048576) GROUP BY \"Http.Domain\", \"Client IP\" ORDER BY \"sessions\" DESC LIMIT 10000\n--Q43.Domain is Accessed by Unique Client IP Trend(bytes Time Grain 5 minute)\nSELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) AS _time , http_domain AS Domain, COUNT(DISTINCT(common_client_ip)) AS nums FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) AND http_domain IN ( SELECT http_domain FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) GROUP BY http_domain ORDER BY SUM(common_s2c_byte_num+common_c2s_byte_num) DESC LIMIT 5 ) GROUP BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) , http_domain ORDER BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) DESC LIMIT 10000\n--Q44. Domain is Accessed by Unique Client IP Trend(sessions,Time Grain 5 minute)\nSELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),3600)*3600) AS stat_time , http_domain , uniq (common_client_ip) AS nums FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start)-604800 AND common_recv_time < toDateTime(@end) AND http_domain IN ( SELECT http_domain FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) GROUP BY http_domain ORDER BY COUNT(*) desc LIMIT 5 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 3600)*3600), http_domain ORDER BY stat_time desc LIMIT 10000\n--Q45.Bandwidth Trend with Device ID(Time Grain 5 minute)\nSELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS \"Receive Time\", common_device_id AS \"Device ID\", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS \"Bytes\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY \"Receive Time\", \"Device ID\" LIMIT 10000\n--Q46.Internal IP by Sled IP and Sessions\nSELECT \"Internal IP\" AS \"Internal IP\", \"Sled IP\" AS \"Sled IP\", sum(coalesce(\"Sessions\", 0)) AS \"Sessions\" FROM (SELECT common_sled_ip AS \"Sled IP\", common_internal_ip AS \"Internal IP\", sum(coalesce(common_sessions, 0)) AS \"Sessions\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY \"Sled IP\", \"Internal IP\" LIMIT 1048576) GROUP BY \"Internal IP\", \"Sled IP\" ORDER BY \"Sessions\" DESC LIMIT 10000\n--Q47.Bandwidth Trend with Internal IP (Time Grain 5 minute)\nSELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS \"Receive Time\", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS \"Bytes\", sum(coalesce(common_c2s_pkt_num + common_s2c_pkt_num, 0)) AS \"Packets\", sum(coalesce(common_sessions, 0)) AS \"New Sessions\", sum(coalesce(common_c2s_byte_num, 0)) AS \"Bytes Sent\", sum(coalesce(common_s2c_byte_num, 0)) AS \"Bytes Received\", sum(coalesce(common_c2s_pkt_num, 0)) AS \"Packets Sent\", sum(coalesce(common_s2c_pkt_num, 0)) AS \"Packets Received\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) GROUP BY \"Receive Time\" LIMIT 10000\n--Q48.Top30 Domains Detail with Internal IP\nSELECT \"Domain\" AS \"Domain\", sum(coalesce(\"Sessions\", 0)) AS \"Sessions\" FROM (SELECT http_domain AS \"Domain\", sum(coalesce(common_sessions, 0)) AS \"Sessions\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) AND ( notEmpty(http_domain) ) GROUP BY \"Domain\" LIMIT 1048576) GROUP BY \"Domain\" ORDER BY \"Sessions\" DESC LIMIT 30\n--Q49.Top30 URLS Detail with Internal IP\nSELECT \"URL\" AS \"URL\", sum(coalesce(\"Sessions\", 0)) AS \"Sessions\" FROM (SELECT http_url AS \"URL\", sum(coalesce(common_sessions, 0)) AS \"Sessions\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) AND ( notEmpty(http_url) ) GROUP BY \"URL\" LIMIT 1048576) GROUP BY \"URL\" ORDER BY \"Sessions\" DESC LIMIT 30\n--Q50.Top Domains with Unique Client IP and Subscriber ID\nSELECT \"Http.Domain\" AS \"Http.Domain\", sum(coalesce(\"Unique Client IP\", 0)) AS \"Unique Client IP\", sum(coalesce(\"Unique Subscriber ID\", 0)) AS \"Unique Subscriber ID\" FROM (SELECT http_domain AS \"Http.Domain\", uniq(common_client_ip) AS \"Unique Client IP\", uniq(common_subscriber_id) AS \"Unique Subscriber ID\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY \"Http.Domain\" LIMIT 1048576) GROUP BY \"Http.Domain\" ORDER BY \"Unique Client IP\" DESC LIMIT 100\n--Q51.Top100 Domains by Packets sent\nSELECT \"Http.Domain\" AS \"Http.Domain\", sum(coalesce(\"Packets Sent\", 0)) AS \"Packets Sent\" FROM (SELECT http_domain AS \"Http.Domain\", sum(coalesce(common_c2s_pkt_num, 0)) AS \"Packets Sent\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY \"Http.Domain\" LIMIT 1048576) GROUP BY \"Http.Domain\" ORDER BY \"Packets Sent\" DESC LIMIT 100\n--Q52.Internal and External asymmetric traffic\nSELECT \"Internal IP\" AS \"Internal IP\", \"External IP\" AS \"External IP\", \"Sled IP\" AS \"Sled IP\", sum(coalesce(\"Sessions\", 0)) AS \"Sessions\" FROM (SELECT common_sled_ip AS \"Sled IP\", common_external_ip AS \"External IP\", common_internal_ip AS \"Internal IP\", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS \"Bytes Sent+Bytes Received\", sum(coalesce(common_sessions, 0)) AS \"Sessions\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_stream_dir != 3 ) GROUP BY \"Sled IP\", \"External IP\", \"Internal IP\" LIMIT 1048576) GROUP BY \"Internal IP\", \"External IP\", \"Sled IP\" ORDER BY \"Sessions\" DESC LIMIT 500\n--Q53.Client and Server ASN asymmetric traffic\nSELECT \"Client ASN\" AS \"Client ASN\", \"Server ASN\" AS \"Server ASN\", sum(coalesce(\"Sessions\", 0)) AS \"Sessions\" FROM (SELECT common_server_asn AS \"Server ASN\", common_client_asn AS \"Client ASN\", sum(coalesce(common_sessions, 0)) AS \"Sessions\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_stream_dir != 3 ) GROUP BY \"Server ASN\", \"Client ASN\" LIMIT 1048576) GROUP BY \"Client ASN\", \"Server ASN\" ORDER BY \"Sessions\" DESC LIMIT 500\n--Q54.Top handshake latency by Website and Client IPs\nSELECT \"SSL.SNI\" AS \"SSL.SNI\", \"Client IP\" AS \"Client IP\", avg(coalesce(\"Establish Latency(ms)\", 0)) AS \"Establish Latency(ms)\" FROM (SELECT common_client_ip AS \"Client IP\", ssl_sni AS \"SSL.SNI\", avg(coalesce(common_establish_latency_ms, 0)) AS \"Establish Latency(ms)\" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY \"Client IP\", \"SSL.SNI\" LIMIT 1048576) GROUP BY \"SSL.SNI\", \"Client IP\" ORDER BY \"Establish Latency(ms)\" DESC LIMIT 500', '8cfc09545177cd1e8ae138917314bbe2', '2021-06-01 21:56:27', '2021-06-01 21:56:27', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'text', NULL); +INSERT INTO `config_info` VALUES (4, 'druid-queries-template.sql', 'Galaxy', '--Q01.某策略命中计数\nselect policy_id, sum(hits) as hits from security_event_hits_log where __time >@start and __time <@end and policy_id=40 group by policy_id\n--Q02.\nselect policy_id, sum(hits) as hits from security_event_hits_log where __time >@start and __time <@end and policy_id in (9,10,88,45) group by policy_id\n--Q03.某策略命中计数趋势\nselect TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as start_time, sum(hits) as hits from security_event_hits_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end and policy_id=10 group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') limit 100\n--Q04.某策略命中计数趋势\nselect DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') as start_time, sum(hits) as hits from security_event_hits_log where __time >= @start and __time < @end and policy_id=10 group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') limit 100\n--Q05.某策略命中时间（首次和最近一次)\nselect policy_id,TIME_FORMAT(min(__time) ,\'yyyy-MM-dd HH:mm:ss\') as first_used, TIME_FORMAT(max(__time) ,\'yyyy-MM-dd HH:mm:ss\') as last_used from security_event_hits_log where policy_id in (100,101 ,105, 102) group by policy_id\n--Q06.\nselect policy_id, DATE_FORMAT(min(__time) ,\'%Y-%m-%d %H:%i:%s\') as first_used, DATE_FORMAT(max(__time) ,\'%Y-%m-%d %H:%i:%s\') as last_used from security_event_hits_log where policy_id in (100,101 ,105, 102) group by policy_id\n--Q07.TopN 命中策略\nselect policy_id, action, sum(hits) as hits from security_event_hits_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by policy_id, action order by hits desc limit 200\n--Q08.\nselect policy_id, action, sum(hits) as hits from security_event_hits_log where __time >=@start and __time <@end group by policy_id, action order by hits desc limit 200\n--Q09.代理策略命中统计 1. 某策略命中计数\nselect policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end and policy_id=100 group by policy_id\n--Q10.\nselect policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end and policy_id=100 group by policy_id\n--Q11.某策略命中趋势\nselect TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as start_time, sum(hits) as hits from proxy_event_hits_log where __time >= TIMESTAMP @start and __time <TIMESTAMP @end and policy_id=100 group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') limit 101\n--Q12.\nselect FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300) as start_time, sum(hits) as hits from proxy_event_hits_log where __time >= @start and __time < @end and policy_id=100 group by FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300) limit 101\n--Q13.某策略命中时间（首次和最近一次\nselect policy_id,TIME_FORMAT(min(__time) ,\'yyyy-MM-dd HH:mm:ss\') as first_used, TIME_FORMAT(max(__time) ,\'yyyy-MM-dd HH:mm:ss\') as last_used from proxy_event_hits_log where policy_id in (100,101,102,105) group by policy_id\n--Q14.\nselect policy_id, DATE_FORMAT(min(__time) ,\'%Y-%m-%d %H:%i:%s\') as first_used, DATE_FORMAT(max(__time) ,\'%Y-%m-%d %H:%i:%s\') as last_used from proxy_event_hits_log where policy_id in (100,101,102,105) group by policy_id\n--Q15. TopN 命中策略\nselect policy_id, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by policy_id, sub_action order by hits desc limit 200\n--Q16. TopN 命中策略\nselect policy_id, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end group by policy_id, sub_action order by hits desc limit 200\n--Q17. Proxy 操纵动作命中计数\nselect sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by sub_action\n--Q18.\nselect sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >= @start and __time < @end group by sub_action\n--Q19. Proxy 操纵动作命中趋势\nselect TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as start_time, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') , sub_action limit 100\n--Q20.\nselect DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') as start_time, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\'), sub_action limit 100\n--Q21. Proxy Pinning TIMESTAMP 计数\nSELECT sum(not_pinning_num) AS sessions, \'notPinningNum\' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end UNION ALL SELECT sum(pinning_num) AS sessions, \'pinningNum\' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end UNION ALL SELECT sum(maybe_pinning_num) AS sessions, \'maybePinningNum\' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end\n--Q22. Proxy Pinning计数趋势\nSELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,\'PT300S\'))/1000)),\'YYYY-MM-dd HH:mm:ss\') AS statisticTime, sum(pinning_num) AS sessions FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,\'PT300S\'))/1000)),\'YYYY-MM-dd HH:mm:ss\') LIMIT 100\n--Q23.\nSELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,\'PT300S\'))/1000)),\'YYYY-MM-dd HH:mm:ss\') AS statisticTime, sum(not_pinning_num) AS sessions FROM traffic_metrics_log WHERE __time>= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,\'PT300S\'))/1000)),\'YYYY-MM-dd HH:mm:ss\') LIMIT 100\n--Q24.\nSELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,\'PT300S\'))/1000)),\'YYYY-MM-dd HH:mm:ss\') AS statisticTime, sum(maybe_pinning_num) AS sessions FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,\'PT300S\'))/1000)),\'YYYY-MM-dd HH:mm:ss\') LIMIT 100\n--Q25.Traffics-带宽统计 1. Traffic IN/OUT 计数 Bytes\nselect sum(total_in_bytes) as traffic_in_bytes, sum(total_out_bytes) as traffic_out_bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end\n--Q26.\nselect sum(total_in_bytes) as traffic_in_bytes, sum(total_out_bytes) as traffic_out_bytes from traffic_metrics_log where __time >= @start and __time < @end\n--Q27. //Packets\nselect sum(total_in_packets) as traffic_in_packets, sum(total_out_packets) as traffic_out_packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end\n--Q28.\nselect sum(total_in_packets) as traffic_in_packets, sum(total_out_packets) as traffic_out_packets from traffic_metrics_log where __time >= @start and __time < @end\n--Q29.//Sessions\nselect sum(new_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end\n--Q30.\nselect sum(new_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end\n--Q31.2. Traffic IN/OUT 带宽趋势 //Bytes\nselect TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'traffic_in_bytes\' as type, sum(total_in_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\') union all select TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'traffic_out_bytes\' as type, sum(total_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\')\n--Q32.\nselect DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'traffic_in_bytes\' as type, sum(total_in_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'traffic_out_bytes\' as type, sum(total_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\')\n--Q33. Packets\nselect TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'traffic_in_packets\' as type, sum(total_in_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\') union all select TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'traffic_out_packets\' as type, sum(total_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\')\n--Q34.\nselect DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'traffic_in_packets\' as type, sum(total_in_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'traffic_out_packets\' as type, sum(total_out_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\')\n--Q35. Packets\nselect TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'traffic_in_packets\' as type, sum(total_in_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\') union all select TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'traffic_out_packets\' as type, sum(total_out_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\')\n--Q36.\nselect DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'traffic_in_packets\' as type, sum(total_in_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'traffic_out_packets\' as type, sum(total_out_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\')\n--Q37.Sessions\nselect TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'new_conn_num\' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\')\n--Q38.\nselect DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'new_conn_num\' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\')\n--Q39.四、 Traffics-计数统计 1. 新建与活跃链接计数\nselect sum(new_conn_num) as new_conn_num, sum(established_conn_num) as established_conn_num from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end\n--Q40.\nselect sum(new_conn_num) as new_conn_num, sum(established_conn_num) as established_conn_num from traffic_metrics_log where __time >= @start and __time < @end\n--Q41. 2.新建与活跃链接计数趋势\nselect TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'new_conn_num\' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\') union all select TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'established_conn_num\' as type, sum(established_conn_num) as sessions from traffic_metrics_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT30S\'),\'yyyy-MM-dd HH:mm:ss\')\n--Q42.\nselect DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'new_conn_num\' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'established_conn_num\' as type, sum(established_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),\'%Y-%m-%d %H:%i:%s\')\n--Q43. 3. 安全策略动作计数 Bytes\nselect sum(default_in_bytes+default_out_bytes) as default_bytes, sum(allow_in_bytes+allow_out_bytes) as allow_bytes, sum(deny_in_bytes+deny_out_bytes) as deny_bytes, sum(monitor_in_bytes+monitor_out_bytes) as monitor_bytes, sum(intercept_in_bytes+intercept_out_bytes) as intercept_bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time < TIMESTAMP @end\n--Q44.\nselect sum(default_in_bytes+default_out_bytes) as default_bytes, sum(allow_in_bytes+allow_out_bytes) as allow_bytes, sum(deny_in_bytes+deny_out_bytes) as deny_bytes, sum(monitor_in_bytes+monitor_out_bytes) as monitor_bytes, sum(intercept_in_bytes+intercept_out_bytes) as intercept_bytes from traffic_metrics_log where __time >= @start and __time < @end\n--Q45.Packets\nselect sum(default_in_packets+default_out_packets) as default_packets, sum(allow_in_packets+allow_in_packets) as allow_packets, sum(deny_in_packets+deny_out_packets) as deny_packets, sum(monitor_in_packets+monitor_out_packets) as monitor_packets, sum(intercept_in_packets+intercept_out_packets) as intercept_packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end\n--Q46.\nselect sum(default_in_packets+default_out_packets) as default_packets, sum(allow_in_packets+allow_in_packets) as allow_packets, sum(deny_in_packets+deny_out_packets) as deny_packets, sum(monitor_in_packets+monitor_out_packets) as monitor_packets, sum(intercept_in_packets+intercept_out_packets) as intercept_packets from traffic_metrics_log where __time >= @start and __time < @end\n--Q47.Sessions\nselect sum(default_conn_num) as default_sessions, sum(allow_conn_num) as allow_sessions, sum(deny_conn_num) as deny_sessions, sum(monitor_conn_num) as monitor_sessions, sum(intercept_conn_num) as intercept_sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end\n--Q48.\nselect sum(default_conn_num) as default_sessions, sum(allow_conn_num) as allow_sessions, sum(deny_conn_num) as deny_sessions, sum(monitor_conn_num) as monitor_sessions, sum(intercept_conn_num) as intercept_sessions from traffic_metrics_log where __time >= @start and __time < @end\n--Q49.4. 安全策略动作趋势 Bytes\nselect TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'default_bytes\' as type, sum(default_in_bytes+default_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') union all select TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'allow_bytes\' as type, sum(allow_in_bytes+allow_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') union all select TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'deny_bytes\' as type, sum(deny_in_bytes+deny_out_bytes) as bytes from traffic_metrics_log where __time >= TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') union all select TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'monitor_bytes\' as type, sum(monitor_in_bytes+monitor_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') union all select TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'intercept_bytes\' as type, sum(intercept_in_bytes+intercept_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\')\n--Q50.\nselect DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'default_bytes\' as type, sum(default_in_bytes+default_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'allow_bytes\' as type, sum(allow_in_bytes+allow_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'deny_bytes\' as type, sum(deny_in_bytes+deny_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'monitor_bytes\' as type, sum(monitor_in_bytes+monitor_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'intercept_bytes\' as type, sum(intercept_in_bytes+intercept_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\')\n--Q51.packets\nselect TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'default_packets\' as type, sum(default_in_packets+default_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') union all select TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'allow_packets\' as type, sum(allow_in_packets+allow_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') union all select TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'deny_packets\' as type, sum(deny_in_packets+deny_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') union all select TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'monitor_packets\' as type, sum(monitor_in_packets+monitor_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') union all select TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'intercept_packets\' as type, sum(intercept_in_packets+intercept_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\')\n--Q52.\nselect DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'default_packets\' as type, sum(default_in_packets+default_out_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'allow_packets\' as type, sum(allow_in_packets+allow_out_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'deny_packets\' as type, sum(deny_in_packets+deny_out_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'monitor_packets\' as type, sum(monitor_in_packets+monitor_out_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'intercept_packets\' as type, sum(intercept_in_packets+intercept_out_packets) as packets from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\')\n--Q53.sessions\nselect TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'default_conn_num\' as type, sum(default_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') union all select TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'allow_conn_num\' as type, sum(allow_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') union all select TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'deny_conn_num\' as type, sum(deny_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') union all select TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'monitor_conn_num\' as type, sum(monitor_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') union all select TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\') as stat_time, \'intercept_conn_num\' as type, sum(intercept_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,\'PT5M\'),\'yyyy-MM-dd HH:mm:ss\')\n--Q54.\nselect DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'default_conn_num\' as type, sum(default_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'allow_conn_num\' as type, sum(allow_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'deny_conn_num\' as type, sum(deny_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'monitor_conn_num\' as type, sum(monitor_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') union all select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\') as stat_time, \'intercept_conn_num\' as type, sum(intercept_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),\'%Y-%m-%d %H:%i:%s\')\n--Q55.五. Traffics-TOPN统计 1. TopN 源IP\nselect source as client_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_internal_host_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by=source group by source order by source desc limit 100\n--Q56.\nselect source as client_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_internal_host_log where __time >= @start and __time < @end and order_by=source group by source order by source desc limit 100\n--Q57. 2. TopN 目的IP\nselect destination as server_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_external_host_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by=destination group by destination order by destination desc limit 100\n--Q58.\nselect destination as server_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_external_host_log where __time >= @start and __time < @end and order_by=destination group by destination order by destination desc limit 100\n--Q59. 3. TopN 域名\nselect domain, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_website_domain_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by=domain group by domain order by domain desc limit 200\n--Q60.\nselect domain, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_website_domain_log where __time >= @start and __time < @end and order_by=domain group by domain order by domain desc limit 200\n--Q61.4.TopN 用户\nselect subscriber_id, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_user_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by=subscriber_id group by subscriber_id order by subscriber_id desc limit 200\n--Q62.\nselect subscriber_id, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_user_log where __time >= @start and __time < @end and order_by=subscriber_id group by subscriber_id order by subscriber_id desc limit 200\n--Q63.六、命中URL统计\nselect url,sum(session_num) as hits from top_urls_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by url order by hits desc limit 100\n--Q64.\nselect url,sum(session_num) as hits from top_urls_log where __time >= @start and __time < @end group by url order by hits desc limit 100\n--Q65.劫持客户端数量\nSELECT policy_id, APPROX_COUNT_DISTINCT_DS_HLL(isp) as num FROM proxy_event_hits_log where __time >= @start and __time < @end and policy_id=0 group by policy_id', 'a243e363ea5631cd56d7ff39334504f5', '2021-06-01 21:56:28', '2021-06-01 21:56:28', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'text', NULL); +INSERT INTO `config_info` VALUES (5, 'engine-queries-template.sql', 'Galaxy', '--Q01.\nselect toDateTime(common_recv_time) as common_recv_time,common_subscriber_id, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, ssl_sni, ssl_version from security_event_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) and common_client_ip like \'49.7%\' order by common_recv_time desc limit 0,20\n--Q02.\nselect FROM_UNIXTIME(common_recv_time) as common_recv_time,common_subscriber_id, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, ssl_sni, ssl_version from security_event_log where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) and common_client_ip like \'49.7%\' order by common_recv_time desc limit 0,20\n--Q03.\nselect toDateTime(common_recv_time) as common_recv_time,common_subscriber_id, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, http_host,http_url,http_user_agent from proxy_event_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) and http_host like \'%joy.cn%\' order by common_recv_time desc limit 0,20\n--Q04.\nselect FROM_UNIXTIME(common_recv_time) as common_recv_time,common_subscriber_id, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, http_host,http_url,http_user_agent from proxy_event_log where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) and http_host like \'%joy.cn%\' order by common_recv_time desc limit 0,20\n--Q05.\nselect toDateTime(common_recv_time) as common_recv_time, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, radius_framed_ip, radius_account from radius_record_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@start) and radius_account=\'T1yRd\' order by common_recv_time desc limit 0,20\n--Q06.\nselect FROM_UNIXTIME(common_recv_time) as common_recv_time,common_subscriber_id, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, http_host,http_url,http_user_agent from proxy_event_log where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) and http_host like \'%joy.cn%\' order by common_recv_time desc limit 0,20\n--Q07.\nselect toDateTime(common_recv_time) as common_recv_time, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, radius_framed_ip, radius_account from radius_record_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@start) and radius_account=\'T1yRd\' order by common_recv_time desc limit 0,20\n--Q08.\nselect FROM_UNIXTIME(common_recv_time) as common_recv_time, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, radius_framed_ip, radius_account from radius_record_log where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) and radius_account=\'T1yRd\' order by common_recv_time desc limit 0,20\n--Q09.\nselect FROM_UNIXTIME(common_recv_time) as common_recv_time,common_log_id,common_policy_id,common_subscriber_id,common_client_ip,common_client_port,common_l4_protocol,common_address_type,common_server_ip,common_server_port,common_action,common_direction,common_sled_ip,common_client_location,common_client_asn,common_server_location,common_server_asn,common_c2s_pkt_num,common_s2c_pkt_num,common_c2s_byte_num,common_s2c_byte_num,common_schema_type,common_sub_action,common_device_id, FROM_UNIXTIME(common_start_time) as common_start_time, FROM_UNIXTIME(common_end_time) as common_end_time,common_establish_latency_ms,common_con_duration_ms,common_stream_dir,common_stream_trace_id,http_url,http_host,http_domain,http_request_body,http_response_body,http_cookie,http_referer,http_user_agent,http_content_length,http_content_type,http_set_cookie,http_version,http_response_lantency_ms,http_action_file_size,http_session_duration_ms,mail_protocol_type,mail_account,mail_from_cmd,mail_to_cmd,mail_from,mail_to,mail_cc,mail_bcc,mail_subject,mail_attachment_name,mail_eml_file,dns_message_id,dns_qr,dns_opcode,dns_aa,dns_tc,dns_rd,dns_ra,dns_rcode,dns_qdcount,dns_ancount,dns_nscount,dns_arcount,dns_qname,dns_qtype,dns_qclass,dns_cname,dns_sub,dns_rr,ssl_sni,ssl_san,ssl_cn,ssl_pinningst,ssl_intercept_state,ssl_server_side_latency,ssl_client_side_latency,ssl_server_side_version,ssl_client_side_version,ssl_cert_verify,ssl_error,quic_version,quic_sni,quic_user_agent,ftp_account,ftp_url,ftp_content from security_event_log where common_recv_time >= @start and common_recv_time < @end and common_log_id in ( 289451559826124800 , 332 ) and ( common_server_ip like \'192.168%\' ) order by common_recv_time desc limit 10000\n--Q10.\nselect count(1) from connection_record_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end)\n--Q11.\nselect count(*) from connection_record_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end)\n--Q12.\nselect * from tsg_galaxy_v3.connection_record_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) limit 1\n--Q13.二次查询\nselect * from (select FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(common_recv_time,\'PT5M\',\'zero\')) as stat_time from connection_record_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) group by stat_time order by stat_time asc)\n--Q14.radius账户申请客户端IP变化\nselect framed_ip, arraySlice(groupUniqArray(concat(toString(event_timestamp),\':\', if(acct_status_type=1,\'start\',\'stop\'))),1,100000) as timeseries from radius_onff_log where event_timestamp >=toDateTime(@start) and event_timestamp <toDateTime(@end) and account=\'admin\' group by framed_ip\n--Q15.\nselect framed_ip, arraySlice(groupUniqArray(concat(toString(event_timestamp),\':\', if(acct_status_type=1,\'start\',\'stop\'))),1,100000) as timeseries from radius_onff_log where event_timestamp >=@start and event_timestamp < @end and account=\'admin\' group by framed_ip\n--Q16.用户IP承载用户变化\nselect account, arraySlice(groupUniqArray(concat(toString(event_timestamp),\':\', if(acct_status_type=1,\'start\',\'stop\'))),1,100000) as timeseries from radius_onff_log where event_timestamp >= @start and event_timestamp < @end and framed_ip=\'127.0.0.1\' group by account\n--Q17.\nselect account, arraySlice(groupUniqArray(concat(toString(event_timestamp),\':\', if(acct_status_type=1,\'start\',\'stop\'))),1,100000) as timeseries from radius_onff_log where event_timestamp >= @start and event_timestamp < @end and framed_ip=\'127.0.0.1\' group by account\n--Q18.预置Internal Hosts 报告\nselect common_client_ip, count(*) as sessions from connection_record_log where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@end)) group by common_client_ip order by sessions desc limit 0,100\n--Q19.预置External Hosts 报告\nselect common_server_ip, count(*) as sessions from connection_record_log where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) group by common_server_ip order by sessions desc limit 0,100\n--Q20.预置Domains报告\nselect http_domain AS domain,SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM connection_record_log WHERE common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(domain) GROUP BY domain ORDER BY bytes DESC LIMIT 100\n--Q21.\nselect toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300) as stat_time, http_domain, uniq (common_client_ip) as nums from connection_record_log where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and http_domain in (select http_domain from connection_record_log where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_domain) group by http_domain order by SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) desc limit 10 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300), http_domain order by stat_time asc limit 500\n--Q22.\nSELECT http_host as host, SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM connection_record_log WHERE common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(http_host) GROUP BY host ORDER BY bytes DESC union all SELECT \'totals\' as host, SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes, SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes, SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes from connection_record_log where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(http_host)\n--Q23.预置HTTP/HTTPS URLS报告\nSELECT http_url AS url,count(*) AS sessions FROM proxy_event_log WHERE common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_url) GROUP BY url ORDER BY sessions DESC LIMIT 100\n--Q24.\nselect toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300) as stat_time, http_url, count(distinct(common_client_ip)) as nums from proxy_event_log where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and http_url IN (select http_url from proxy_event_log where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_url) group by http_url order by count(*) desc limit 10 )group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300), http_url order by stat_time asc limit 500\n--Q25.\nselect common_subscriber_id as user, count(*) as sessions from connection_record_log where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(user) group by common_subscriber_id order by sessions desc limit 0,100\n--Q26.\nSELECT common_subscriber_id as user,SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM connection_record_log WHERE common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(user) GROUP BY user ORDER BY bytes DESC LIMIT 100\n--Q27.RADIUS账户总计\nselect count(distinct(framed_ip)) as active_ip_num , sum(acct_session_time) as online_duration from (select any(framed_ip) as framed_ip ,max(acct_session_time) as acct_session_time from radius_onff_log where account=\'000jS\' and event_timestamp >= @start and event_timestamp < @end group by acct_session_id)\n--Q28.RADIUS账户IP详情\nselect distinct(framed_ip) as framed_ip from radius_onff_log where account=\'000iS\' and event_timestamp >= @start and event_timestamp < @end\n--Q29.RADIUS账户访问详情\nselect max(if(acct_status_type=1,event_timestamp,0)) as start_time,max(if(acct_status_type=2,event_timestamp,0)) as end_time, any(framed_ip) as ip,max(acct_session_time) as online_duration from radius_onff_log where account=\'000jS\' and event_timestamp >= @start and event_timestamp < @end group by acct_session_id order by start_time desc limit 200\n--Q30.自定义函数测试\nSELECT policy_id, APPROX_COUNT_DISTINCT_DS_HLL(isp) as num FROM proxy_event_hits_log where __time >= @start and __time < @end and policy_id=0 group by policy_id\n--Q31.\nselect TIME_FLOOR_WITH_FILL(common_recv_time,\'PT5M\',\'previous\') as stat_time from connection_record_log where common_recv_time > @start and common_recv_time < @end group by stat_time\n--Q32.特殊字符验证\nselect * from connection_record_log where (common_protocol_label =\'/$\' or common_client_ip like\'%\') limit 10', 'ed07b49869f651e3f181c9653e6f1ffa', '2021-06-01 21:56:29', '2021-06-01 21:56:29', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'text', NULL); +INSERT INTO `config_info` VALUES (6, 'active_defence_event_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"active_defence_event_log\",\n \"namespace\": \"tsg_galaxy_v3\",\n \"doc\": {\n \"primary_key\": \"common_log_id\",\n \"partition_key\": \"common_recv_time\",\n \"schema_query\": {\n \"dimensions\": [\n \"common_policy_id\",\n \"ad_target_ip\",\n \"ad_cc_target_url\"\n ],\n \"metrics\": [\n \"ad_target_ip\",\n \"ad_sent_byte_num\",\n \"ad_sent_pkt_num\",\n \"ad_cc_initiate_connection_num\",\n \"ad_cc_established_connection_num\",\n \"ad_cc_rejected_connection_num\"\n ],\n \"filters\": [\n \"common_policy_id\",\n \"ad_target_ip\",\n \"ad_target_port\",\n \"ad_protocol\",\n \"common_address_type\",\n \"ad_sent_byte_num\",\n \"ad_sent_pkt_num\",\n \"ad_cc_initiate_connection_num\",\n \"ad_cc_established_connection_num\",\n \"ad_cc_rejected_connection_num\"\n ]\n },\n \"schema_type\": {\n \"REFLECTION\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_address_type\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_entrance_id\",\n \"common_user_region\",\n \"ad_method\",\n \"ad_protocol\",\n \"ad_target_ip\",\n \"ad_target_port\",\n \"ad_target_ip_location\",\n \"ad_target_ip_asn\",\n \"ad_reflector_profile_id\",\n \"ad_sent_pkt_num\",\n \"ad_sent_byte_num\",\n \"ad_generate_time\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"ad_target_ip\",\n \"ad_target_port\",\n \"ad_reflector_profile_id\",\n \"ad_sent_pkt_num\",\n \"ad_sent_byte_num\"\n ]\n },\n \"FLOOD\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_address_type\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_entrance_id\",\n \"common_user_region\",\n \"ad_method\",\n \"ad_protocol\",\n \"ad_target_ip\",\n \"ad_target_port\",\n \"ad_target_ip_location\",\n \"ad_target_ip_asn\",\n \"ad_claimed_src_ip_profile_id\",\n \"ad_sent_pkt_num\",\n \"ad_sent_byte_num\",\n \"ad_generate_time\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"ad_target_ip\",\n \"ad_target_port\",\n \"ad_claimed_src_ip_profile_id\",\n \"ad_protocol\"\n ]\n },\n \"CC\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_address_type\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_entrance_id\",\n \"common_user_region\",\n \"ad_method\",\n \"ad_protocol\",\n \"ad_cc_target_url\",\n \"ad_claimed_src_ip_profile_id\",\n \"ad_cc_initiate_connection_num\",\n \"ad_cc_established_connection_num\",\n \"ad_cc_rejected_connection_num\",\n \"ad_generate_time\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"ad_cc_target_url\",\n \"ad_claimed_src_ip_profile_id\",\n \"ad_protocol\"\n ]\n }\n },\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"ad_target_ip\",\n \"ad_target_port\",\n \"ad_cc_target_url\"\n ]\n },\n \"fields\": [\n {\n \"name\": \"common_recv_time\",\n \"label\": \"Receive Time\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"current_timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_log_id\",\n \"label\": \"Log ID\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"snowflake_id\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_policy_id\",\n \"label\": \"Policy ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_address_type\",\n \"label\": \"Address Type\",\n \"doc\": {\n \"data\": [\n {\n \"code\": \"4\",\n \"value\": \"ipv4\"\n },\n {\n \"code\": \"6\",\n \"value\": \"ipv6\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_entrance_id\",\n \"label\": \"Entrance ID\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_device_id\",\n \"label\": \"Device ID\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_link_id\",\n \"label\": \"Link ID\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_user_region\",\n \"label\": \"User Region\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ad_target_ip\",\n \"label\": \"Target IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"geo_ip_country,geo_asn\",\n \"appendTo\": \"ad_target_ip_location,ad_target_ip_asn\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ad_target_port\",\n \"label\": \"Target Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"ad_cc_target_url\",\n \"label\": \"Target URL\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ad_target_ip_location\",\n \"label\": \"Target Location\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ad_target_ip_asn\",\n \"label\": \"Target ASN\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ad_protocol\",\n \"label\": \"Protocol\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ad_method\",\n \"label\": \"Method\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ad_claimed_src_ip_profile_id\",\n \"label\": \"Claimed Profile ID\",\n \"type\": \"int\"\n },\n {\n \"name\": \"ad_reflector_profile_id\",\n \"label\": \"Reflector Profile ID\",\n \"type\": \"int\"\n },\n {\n \"name\": \"ad_sent_pkt_num\",\n \"label\": \"Packets Sent\",\n \"type\": \"int\"\n },\n {\n \"name\": \"ad_sent_byte_num\",\n \"label\": \"Bytes Sent\",\n \"type\": \"int\"\n },\n {\n \"name\": \"ad_cc_initiate_connection_num\",\n \"label\": \"Initiate Numbers\",\n \"type\": \"int\"\n },\n {\n \"name\": \"ad_cc_established_connection_num\",\n \"label\": \"Established Numbers\",\n \"type\": \"int\"\n },\n {\n \"name\": \"ad_cc_rejected_connection_num\",\n \"label\": \"Rejected Numbers\",\n \"type\": \"int\"\n },\n {\n \"name\": \"ad_generate_time\",\n \"label\": \"Generate Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n }\n },\n \"type\": \"int\"\n }\n ]\n}', 'b27f4f9e685ddf0040ba5957b1340d03', '2021-06-01 21:56:31', '2021-06-01 21:56:31', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (7, 'ck-filter.json', 'Galaxy', '{\n \"version\": \"1.0\",\n \"name\": \"connection_record_log\",\n \"namespace\": \"Clickhouse\",\n \"filters\": [\n {\n \"name\":\"@start\",\n \"value\": \"\'2021-05-06 00:00:00\'\"\n },\n {\n \"name\":\"@end\",\n \"value\": \"\'2021-05-07 00:00:00\'\"\n },\n {\n \"name\":\"@common_filter\",\n \"value\": [\n \"common_client_ip=\'192.168.44.21\'and common_server_port=443\",\n \"common_server_ip=\'8.8.8.8\' and common_server_port=443\",\n \"http_domain like \'%baidu%\' and common_server_port=443\",\n \"http_domain like \'%baidu%\'and common_server_port=443\"\n ]\n },\n {\n \"name\":\"@index_filter\",\n \"value\": [\n \"common_server_ip=\'8.8.8.8\'\",\n \"common_client_ip=\'192.168.44.21\'\",\n \"http_domain like \'%baidu%\'\",\n \"common_sled_ip=\'%192.168%\'\",\n \"common_subscriber_id=\'%test%\'\"\n ]\n }\n ]\n}', '800dcd30706662dc298fffb35623ac53', '2021-06-01 21:56:31', '2021-06-01 21:56:31', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (8, 'columns_cluster.json', 'Galaxy', '{\n \"namespace\": \"system\",\n \"type\": \"record\",\n \"name\": \"columns_cluster\",\n \"fields\": [\n {\n \"name\": \"database\",\n \"type\": \"string\"\n }\n ]\n}', '4532c0de2639d00c5ad98ab801cae6bb', '2021-06-01 21:56:32', '2021-06-01 21:56:32', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (9, 'connection_record_log_common_client_ip.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"connection_record_log_common_client_ip\",\n \"namespace\": \"tsg_galaxy_v3\",\n \"doc\": {\n \"primary_key\": \"common_log_id\",\n \"index_key\": \"common_client_ip\"\n },\n \"fields\": [\n {\n \"name\": \"common_log_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_recv_time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_policy_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_action\",\n \"type\": \"int\"\n },\n {\n \"name\": \"common_server_ip\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_ip\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sled_ip\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_entrance_id\",\n \"type\": \"int\"\n },\n {\n \"name\": \"common_subscriber_id\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_stream_trace_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"http_domain\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_sni\",\n \"type\": \"string\"\n }\n ]\n}', 'aa81f9d20718f7f82fa418e9740e5fc7', '2021-06-01 21:56:33', '2021-06-01 21:56:33', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (10, 'connection_record_log_common_server_ip.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"connection_record_log_common_server_ip\",\n \"namespace\": \"tsg_galaxy_v3\",\n \"doc\": {\n \"primary_key\": \"common_log_id\",\n \"index_key\": \"common_server_ip\"\n },\n \"fields\": [\n {\n \"name\": \"common_log_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_recv_time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_policy_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_action\",\n \"type\": \"int\"\n },\n {\n \"name\": \"common_server_ip\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_ip\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sled_ip\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_entrance_id\",\n \"type\": \"int\"\n },\n {\n \"name\": \"common_subscriber_id\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_stream_trace_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"http_domain\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_sni\",\n \"type\": \"string\"\n }\n ]\n}', 'c8592e44fa18831cc5094a6f8467eaff', '2021-06-01 21:56:34', '2021-06-01 21:56:34', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (11, 'connection_record_log_common_subscriber_id.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"connection_record_log_common_subscriber_id\",\n \"namespace\": \"tsg_galaxy_v3\",\n \"doc\": {\n \"primary_key\": \"common_log_id\",\n \"index_key\": \"common_subscriber_id\"\n },\n \"fields\": [\n {\n \"name\": \"common_log_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_recv_time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_policy_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_action\",\n \"type\": \"int\"\n },\n {\n \"name\": \"common_server_ip\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_ip\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sled_ip\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_entrance_id\",\n \"type\": \"int\"\n },\n {\n \"name\": \"common_subscriber_id\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_stream_trace_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"http_domain\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_sni\",\n \"type\": \"string\"\n }\n ]\n}', '1df2e9730baf7952c34ad0fa95d27268', '2021-06-01 21:56:34', '2021-06-01 21:56:34', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (12, 'connection_record_log_http_domain.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"connection_record_log_http_domain\",\n \"namespace\": \"tsg_galaxy_v3\",\n \"doc\": {\n \"primary_key\": \"common_log_id\",\n \"index_key\": \"http_domain\"\n },\n \"fields\": [\n {\n \"name\": \"common_log_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_recv_time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_policy_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_action\",\n \"type\": \"int\"\n },\n {\n \"name\": \"common_server_ip\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_ip\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sled_ip\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_entrance_id\",\n \"type\": \"int\"\n },\n {\n \"name\": \"common_subscriber_id\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_stream_trace_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"http_domain\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_sni\",\n \"type\": \"string\"\n }\n ]\n}', '4cf16b4936e6aafe04035da82a7e2767', '2021-06-01 21:56:35', '2021-06-01 21:56:35', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (13, 'connection_record_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"connection_record_log\",\n \"namespace\": \"tsg_galaxy_v3\",\n \"doc\": {\n \"primary_key\": \"common_log_id\",\n \"partition_key\": \"common_recv_time\",\n \"index_table\": \"connection_record_log_common_client_ip,connection_record_log_common_server_ip,connection_record_log_common_subscriber_id,connection_record_log_http_domain\",\n \"functions\": {\n \"aggregation\": [\n {\n \"name\": \"COUNT\",\n \"label\": \"COUNT\",\n \"function\": \"count(expr)\"\n },\n {\n \"name\": \"COUNT_DISTINCT\",\n \"label\": \"COUNT_DISTINCT\",\n \"function\": \"count(distinct expr)\"\n },\n {\n \"name\": \"AVG\",\n \"label\": \"AVG\",\n \"function\": \"avg(expr)\"\n },\n {\n \"name\": \"SUM\",\n \"label\": \"SUM\",\n \"function\": \"sum(expr)\"\n },\n {\n \"name\": \"MAX\",\n \"label\": \"MAX\",\n \"function\": \"max(expr)\"\n },\n {\n \"name\": \"MIN\",\n \"label\": \"MIN\",\n \"function\": \"min(expr)\"\n }\n ],\n \"operator\": [\n {\n \"name\": \"=\",\n \"label\": \"=\",\n \"function\": \"expr = value\"\n },\n {\n \"name\": \"!=\",\n \"label\": \"!=\",\n \"function\": \"expr != value\"\n },\n {\n \"name\": \">\",\n \"label\": \">\",\n \"function\": \"expr > value\"\n },\n {\n \"name\": \"<\",\n \"label\": \"<\",\n \"function\": \"expr < value\"\n },\n {\n \"name\": \">=\",\n \"label\": \">=\",\n \"function\": \"expr >= value\"\n },\n {\n \"name\": \"<=\",\n \"label\": \"<=\",\n \"function\": \"expr <= value\"\n },\n {\n \"name\": \"in\",\n \"label\": \"IN\",\n \"function\": \"expr in (values)\"\n },\n {\n \"name\": \"not in\",\n \"label\": \"NOT IN\",\n \"function\": \"expr not in (values)\"\n },\n {\n \"name\": \"like\",\n \"label\": \"LIKE\",\n \"function\": \"expr like value\"\n },\n {\n \"name\": \"not like\",\n \"label\": \"NOT LIKE\",\n \"function\": \"expr not like value\"\n },\n {\n \"name\": \"notEmpty\",\n \"label\": \"NOT EMPTY\",\n \"function\": \"notEmpty(expr)\"\n },\n {\n \"name\": \"empty\",\n \"label\": \"EMPTY\",\n \"function\": \"empty(expr)\"\n }\n ]\n },\n \"schema_query\": {\n \"dimensions\": [\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_client_location\",\n \"common_server_location\",\n \"common_subscriber_id\",\n \"common_client_port\",\n \"common_server_port\",\n \"common_schema_type\",\n \"common_l4_protocol\",\n \"common_l7_protocol\",\n \"common_data_center\",\n \"common_client_asn\",\n \"common_server_asn\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"http_host\",\n \"http_domain\",\n \"http_url\",\n \"ssl_sni\",\n \"ssl_ja3_hash\"\n ],\n \"metrics\": [\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_subscriber_id\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_sessions\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"http_host\",\n \"http_domain\",\n \"http_url\",\n \"ssl_sni\",\n \"ssl_ja3_hash\"\n ],\n \"filters\": [\n \"common_address_type\",\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_client_port\",\n \"common_server_port\",\n \"common_client_location\",\n \"common_server_location\",\n \"common_subscriber_id\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_l4_protocol\",\n \"common_l7_protocol\",\n \"common_stream_dir\",\n \"common_direction\",\n \"common_data_center\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_schema_type\",\n \"common_client_asn\",\n \"common_server_asn\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"http_host\",\n \"http_domain\",\n \"http_url\",\n \"ssl_sni\",\n \"ssl_ja3_hash\"\n ],\n \"references\": {\n \"aggregation\": [\n {\n \"type\": \"int\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"long\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"float\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"double\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"string\",\n \"functions\": \"COUNT,COUNT_DISTINCT\"\n },\n {\n \"type\": \"date\",\n \"functions\": \"COUNT,COUNT_DISTINCT,MAX,MIN\"\n },\n {\n \"type\": \"timestamp\",\n \"functions\": \"COUNT,COUNT_DISTINCT,MAX,MIN\"\n }\n ],\n \"operator\": [\n {\n \"type\": \"int\",\n \"functions\": \"=,!=,>,<,>=,<=,in,not in\"\n },\n {\n \"type\": \"long\",\n \"functions\": \"=,!=,>,<,>=,<=,in,not in\"\n },\n {\n \"type\": \"float\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"double\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"string\",\n \"functions\": \"=,!=,in,not in,like,not like,notEmpty,empty\"\n },\n {\n \"type\": \"date\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"timestamp\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n }\n ]\n },\n \"details\": {\n \"general\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_stream_trace_id\",\n \"common_direction\",\n \"common_stream_dir\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_processing_time\",\n \"common_entrance_id\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_data_center\",\n \"common_sled_ip\"\n ],\n \"action\": [\n \"common_action\",\n \"common_sub_action\",\n \"common_policy_id\",\n \"common_user_tags\",\n \"common_user_region\"\n ],\n \"source\": [\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_client_port\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\"\n ],\n \"destination\": [\n \"common_server_ip\",\n \"common_external_ip\",\n \"common_server_port\",\n \"common_server_location\",\n \"common_server_asn\"\n ],\n \"application\": [\n \"common_app_id\",\n \"common_app_label\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_protocol_label\",\n \"common_service_category\",\n \"common_service\",\n \"common_l4_protocol\"\n ],\n \"transmission\": [\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_first_ttl\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\"\n ],\n \"other\": [\n \"common_address_type\",\n \"common_schema_type\",\n \"common_device_tag\",\n \"common_encapsulation\",\n \"common_tunnels\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\"\n ]\n }\n },\n \"schema_type\": {\n \"BASE\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_service_category\",\n \"common_l7_protocol\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"HTTP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_service_category\",\n \"common_l7_protocol\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"http_url\",\n \"http_host\",\n \"http_domain\",\n \"http_request_line\",\n \"http_response_line\",\n \"http_request_header\",\n \"http_response_header\",\n \"http_request_content\",\n \"http_response_content\",\n \"http_request_body\",\n \"http_response_body\",\n \"http_request_body_key\",\n \"http_response_body_key\",\n \"http_proxy_flag\",\n \"http_sequence\",\n \"http_snapshot\",\n \"http_cookie\",\n \"http_referer\",\n \"http_user_agent\",\n \"http_content_length\",\n \"http_content_type\",\n \"http_set_cookie\",\n \"http_version\",\n \"http_response_lantency_ms\",\n \"http_session_duration_ms\",\n \"http_action_file_size\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"http_url\",\n \"common_server_port\"\n ]\n },\n \"MAIL\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"mail_protocol_type\",\n \"mail_account\",\n \"mail_from_cmd\",\n \"mail_to_cmd\",\n \"mail_from\",\n \"mail_to\",\n \"mail_cc\",\n \"mail_bcc\",\n \"mail_subject\",\n \"mail_subject_charset\",\n \"mail_content\",\n \"mail_content_charset\",\n \"mail_attachment_name\",\n \"mail_attachment_name_charset\",\n \"mail_attachment_content\",\n \"mail_eml_file\",\n \"mail_snapshot\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"mail_from\",\n \"mail_to\",\n \"mail_subject\"\n ]\n },\n \"DNS\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"dns_message_id\",\n \"dns_qr\",\n \"dns_opcode\",\n \"dns_aa\",\n \"dns_tc\",\n \"dns_rd\",\n \"dns_ra\",\n \"dns_rcode\",\n \"dns_qdcount\",\n \"dns_ancount\",\n \"dns_nscount\",\n \"dns_arcount\",\n \"dns_qname\",\n \"dns_qtype\",\n \"dns_qclass\",\n \"dns_cname\",\n \"dns_sub\",\n \"dns_rr\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_client_ip\",\n \"dns_qr\",\n \"dns_qname\",\n \"dns_qtype\"\n ]\n },\n \"SSL\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"ssl_sni\",\n \"ssl_san\",\n \"ssl_cn\",\n \"ssl_pinningst\",\n \"ssl_intercept_state\",\n \"ssl_server_side_latency\",\n \"ssl_client_side_latency\",\n \"ssl_server_side_version\",\n \"ssl_client_side_version\",\n \"ssl_cert_verify\",\n \"ssl_error\",\n \"ssl_con_latency_ms\",\n \"ssl_ja3_fingerprint\",\n \"ssl_ja3_hash\",\n \"ssl_cert_issuer\",\n \"ssl_cert_subject\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"ssl_sni\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"QUIC\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"quic_version\",\n \"quic_sni\",\n \"quic_user_agent\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"quic_sni\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"FTP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"ftp_account\",\n \"ftp_url\",\n \"ftp_content\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"ftp_url\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"BGP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"bgp_type\",\n \"bgp_as_num\",\n \"bgp_route\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"bgp_type\",\n \"bgp_as_num\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"SIP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"sip_call_id\",\n \"sip_originator_description\",\n \"sip_responder_description\",\n \"sip_user_agent\",\n \"sip_server\",\n \"sip_originator_sdp_connect_ip\",\n \"sip_originator_sdp_media_port\",\n \"sip_originator_sdp_media_type\",\n \"sip_originator_sdp_content\",\n \"sip_responder_sdp_connect_ip\",\n \"sip_responder_sdp_media_port\",\n \"sip_responder_sdp_media_type\",\n \"sip_responder_sdp_content\",\n \"sip_duration\",\n \"sip_bye\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"sip_originator_description\",\n \"sip_responder_description\",\n \"sip_call_id\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"RTP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"rtp_payload_type_c2s\",\n \"rtp_payload_type_s2c\",\n \"rtp_pcap_path\",\n \"rtp_originator_dir\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\",\n \"rtp_pcap_path\"\n ]\n },\n \"APP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"app_extra_info\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_app_id\",\n \"common_app_label\",\n \"app_extra_info\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n }\n },\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_schema_type\"\n ],\n \"tunnel_type\": {\n \"GTP\": [\n {\n \"name\": \"gtp_sgw_ip\",\n \"label\": \"S-GW IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_pgw_ip\",\n \"label\": \"P-GW IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_sgw_port\",\n \"label\": \"S-GW Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"gtp_pgw_port\",\n \"label\": \"P-GW Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"gtp_uplink_teid\",\n \"label\": \"Uplink TEID\",\n \"type\": \"long\"\n },\n {\n \"name\": \"gtp_downlink_teid\",\n \"label\": \"Downlink TEID\",\n \"type\": \"long\"\n }\n ],\n \"MPLS\": [\n {\n \"name\": \"mpls_c2s_direction_label\",\n \"label\": \"Multiprotocol Label(c2s)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n },\n {\n \"name\": \"mpls_s2c_direction_label\",\n \"label\": \"Multiprotocol Label(s2c)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n }\n ],\n \"VLAN\": [\n {\n \"name\": \"vlan_c2s_direction_id\",\n \"label\": \"VLAN Direction(c2s)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n },\n {\n \"name\": \"vlan_s2c_direction_id\",\n \"label\": \"VLAN Direction(s2c)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n }\n ],\n \"ETHERNET\": [\n {\n \"name\": \"source_mac\",\n \"label\": \"Source MAC\",\n \"type\": \"string\"\n },\n {\n \"name\": \"destination_mac\",\n \"label\": \"Destination MAC\",\n \"type\": \"string\"\n }\n ],\n \"MULTIPATH_ETHERNET\": [\n {\n \"name\": \"c2s_source_mac\",\n \"label\": \"Source MAC(c2s)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"c2s_destination_mac\",\n \"label\": \"Destination MAC(c2s)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_source_mac\",\n \"label\": \"Source MAC(s2c)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_destination_mac\",\n \"label\": \"Destination MAC(s2c)\",\n \"type\": \"string\"\n }\n ]\n }\n },\n \"fields\": [\n {\n \"name\": \"common_recv_time\",\n \"label\": \"Receive Time\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_log_id\",\n \"label\": \"Log ID\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"snowflake_id\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_policy_id\",\n \"label\": \"Policy ID\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_subscriber_id\",\n \"label\": \"Subscriber ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_imei\",\n \"label\": \"IMEI\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_imsi\",\n \"label\": \"IMSI\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_phone_number\",\n \"label\": \"Phone Number\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_ip\",\n \"label\": \"Client IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"geo_asn,radius_match\",\n \"appendTo\": \"common_client_asn,common_subscriber_id\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_internal_ip\",\n \"label\": \"Internal IP\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"if\",\n \"param\": \"$.common_direction=69,$.common_client_ip,$.common_server_ip\"\n },\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_port\",\n \"label\": \"Client Port\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_l4_protocol\",\n \"label\": \"L4 Protocol\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_address_type\",\n \"label\": \"Address Type\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"4\",\n \"value\": \"ipv4\"\n },\n {\n \"code\": \"6\",\n \"value\": \"ipv6\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_server_ip\",\n \"label\": \"Server IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"geo_asn\",\n \"appendTo\": \"common_server_asn\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_port\",\n \"label\": \"Server Port\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_external_ip\",\n \"label\": \"External IP\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"if\",\n \"param\": \"$.common_direction=73,$.common_client_ip,$.common_server_ip\"\n },\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_action\",\n \"label\": \"Action\",\n \"doc\": {\n \"visibility\": \"hidden\",\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"None\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Intercept\"\n },\n {\n \"code\": \"16\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"128\",\n \"value\": \"Allow\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_direction\",\n \"label\": \"Direction\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"69\",\n \"value\": \"outbound\"\n },\n {\n \"code\": \"73\",\n \"value\": \"inbound\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_entrance_id\",\n \"label\": \"Entrance ID\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_sled_ip\",\n \"label\": \"Sled IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_location\",\n \"label\": \"Client Location\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_asn\",\n \"label\": \"Client ASN\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_location\",\n \"label\": \"Server Location\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_asn\",\n \"label\": \"Server ASN\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sessions\",\n \"label\": \"Sessions\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_num\",\n \"label\": \"Packets Sent\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_num\",\n \"label\": \"Packets Received\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_num\",\n \"label\": \"Bytes Sent\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_num\",\n \"label\": \"Bytes Received\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_diff\",\n \"label\": \"Packets Sent(Diff)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_diff\",\n \"label\": \"Packets Received(Diff)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_diff\",\n \"label\": \"Bytes Sent(Diff)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_diff\",\n \"label\": \"Bytes Received(Diff)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_service\",\n \"label\": \"Service\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_schema_type\",\n \"label\": \"Schema Type\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"BASE\",\n \"value\": \"BASE\"\n },\n {\n \"code\": \"MAIL\",\n \"value\": \"MAIL\"\n },\n {\n \"code\": \"DNS\",\n \"value\": \"DNS\"\n },\n {\n \"code\": \"HTTP\",\n \"value\": \"HTTP\"\n },\n {\n \"code\": \"SSL\",\n \"value\": \"SSL\"\n },\n {\n \"code\": \"FTP\",\n \"value\": \"FTP\"\n },\n {\n \"code\": \"SIP\",\n \"value\": \"SIP\"\n },\n {\n \"code\": \"RTP\",\n \"value\": \"RTP\"\n },\n {\n \"code\": \"APP\",\n \"value\": \"APP\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_user_tags\",\n \"label\": \"User Tags\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sub_action\",\n \"label\": \"Sub Action\",\n \"doc\": {\n \"data\": [\n {\n \"code\": \"allow\",\n \"value\": \"Allow\"\n },\n {\n \"code\": \"deny\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"monitor\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"replace\",\n \"value\": \"Replace\"\n },\n {\n \"code\": \"redirect\",\n \"value\": \"Redirect\"\n },\n {\n \"code\": \"insert\",\n \"value\": \"Insert\"\n },\n {\n \"code\": \"hijack\",\n \"value\": \"Hijack\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_user_region\",\n \"label\": \"User Region\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_device_id\",\n \"label\": \"Device ID\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_link_id\",\n \"label\": \"Link ID\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_isp\",\n \"label\": \"ISP\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_device_tag\",\n \"label\": \"Device Tag\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_data_center\",\n \"label\": \"Data Center\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"City A\",\n \"value\": \"City A\"\n },\n {\n \"code\": \"City B\",\n \"value\": \"City B\"\n },\n {\n \"code\": \"City C\",\n \"value\": \"City C\"\n },\n {\n \"code\": \"City D\",\n \"value\": \"City D\"\n },\n {\n \"code\": \"City E\",\n \"value\": \"City E\"\n },\n {\n \"code\": \"City F\",\n \"value\": \"City F\"\n },\n {\n \"code\": \"City G\",\n \"value\": \"City G\"\n },\n {\n \"code\": \"City H\",\n \"value\": \"City H\"\n },\n {\n \"code\": \"City I\",\n \"value\": \"City I\"\n },\n {\n \"code\": \"City J\",\n \"value\": \"City J\"\n },\n {\n \"code\": \"City K\",\n \"value\": \"City K\"\n },\n {\n \"code\": \"City L\",\n \"value\": \"City L\"\n },\n {\n \"code\": \"City M\",\n \"value\": \"City M\"\n },\n {\n \"code\": \"City N\",\n \"value\": \"City N\"\n },\n {\n \"code\": \"City O\",\n \"value\": \"City O\"\n },\n {\n \"code\": \"City P\",\n \"value\": \"City P\"\n },\n {\n \"code\": \"City Q\",\n \"value\": \"City Q\"\n },\n {\n \"code\": \"City R\",\n \"value\": \"City R\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_encapsulation\",\n \"label\": \"Encapsulation\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"Ethernet\"\n },\n {\n \"code\": \"8\",\n \"value\": \"PPP\"\n },\n {\n \"code\": \"12\",\n \"value\": \"CiscoHDLC\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_app_label\",\n \"label\": \"Application Label\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_tunnels\",\n \"label\": \"Tunnels\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_protocol_label\",\n \"label\": \"Protocol Label\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_app_id\",\n \"label\": \"Application ID\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_app_surrogate_id\",\n \"label\": \"Surrogate ID\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_l7_protocol\",\n \"label\": \"L7 Protocol\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_service_category\",\n \"label\": \"FQDN Category\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"dict_location\": {\n \"path\": \"/v1/category/dict\",\n \"key\": \"categoryId\",\n \"value\": \"categoryName\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_start_time\",\n \"label\": \"Start Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_end_time\",\n \"label\": \"End Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"get_value\",\n \"appendTo\": \"common_recv_time\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_establish_latency_ms\",\n \"label\": \"Establish Latency(ms)\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_con_duration_ms\",\n \"label\": \"Duration(ms)\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_stream_dir\",\n \"label\": \"Stream Direction\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"c2s\"\n },\n {\n \"code\": \"2\",\n \"value\": \"s2c\"\n },\n {\n \"code\": \"3\",\n \"value\": \"double\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_address_list\",\n \"label\": \"Address List\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_has_dup_traffic\",\n \"label\": \"Duplication Traffic\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"No\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Yes\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_stream_error\",\n \"label\": \"Stream Error\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_stream_trace_id\",\n \"label\": \"Session ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_link_info_c2s\",\n \"label\": \"Link Info(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_link_info_s2c\",\n \"label\": \"Link Info(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_c2s_ipfrag_num\",\n \"label\": \"Fragmentation Packets(c2s)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_ipfrag_num\",\n \"label\": \"Fragmentation Packets(s2c)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_tcp_lostlen\",\n \"label\": \"Sequence Gap Loss(c2s)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_tcp_lostlen\",\n \"label\": \"Sequence Gap Loss(s2c)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_tcp_unorder_num\",\n \"label\": \"Unorder Packets(c2s)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_tcp_unorder_num\",\n \"label\": \"Unorder Packets(s2c)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_retrans\",\n \"label\": \"Packet Retransmission(c2s)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_retrans\",\n \"label\": \"Packet Retransmission(s2c)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_retrans\",\n \"label\": \"Byte Retransmission(c2s)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_retrans\",\n \"label\": \"Byte Retransmission(s2c)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_tcp_client_isn\",\n \"label\": \"TCP Client ISN\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_tcp_server_isn\",\n \"label\": \"TCP Server ISN\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_first_ttl\",\n \"label\": \"First TTL\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_processing_time\",\n \"label\": \"Processing Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"current_timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"http_url\",\n \"label\": \"HTTP.URL\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_host\",\n \"label\": \"HTTP.Host\",\n \"doc\": {\n \"format\": {\n \"functions\": \"sub_domain\",\n \"appendTo\": \"http_domain\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_domain\",\n \"label\": \"HTTP.Domain\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_line\",\n \"label\": \"HTTP.Request Line\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_line\",\n \"label\": \"HTTP.Response Line\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_header\",\n \"label\": \"HTTP.Request Headers\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_header\",\n \"label\": \"HTTP.Response Headers\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_content\",\n \"label\": \"HTTP.Request Content\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_content\",\n \"label\": \"HTTP.Response Content\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_body\",\n \"label\": \"HTTP.Request Body\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"file\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_body\",\n \"label\": \"HTTP.Response Body\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"file\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_body_key\",\n \"label\": \"HTTP.Request Body Key\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_body_key\",\n \"label\": \"HTTP.Response Body Key\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_proxy_flag\",\n \"label\": \"HTTP.Proxy Flag\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"http_sequence\",\n \"label\": \"HTTP.Sequence\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"http_snapshot\",\n \"label\": \"HTTP.Snapshot\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_cookie\",\n \"label\": \"HTTP.Cookie\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_referer\",\n \"label\": \"HTTP.Referer\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_user_agent\",\n \"label\": \"HTTP.User Agent\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_content_length\",\n \"label\": \"HTTP.Content Length\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_content_type\",\n \"label\": \"HTTP.Content Type\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_set_cookie\",\n \"label\": \"HTTP.Set Cookie\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_version\",\n \"label\": \"HTTP.Version\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_lantency_ms\",\n \"label\": \"HTTP.Response Latency(ms)\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"http_session_duration_ms\",\n \"label\": \"HTTP.Session Duration(ms)\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"http_action_file_size\",\n \"label\": \"HTTP.Action File Size\",\n \"type\": \"int\"\n },\n {\n \"name\": \"mail_protocol_type\",\n \"label\": \"Mail.Protocol Type\",\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_account\",\n \"label\": \"Mail.Account\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_from_cmd\",\n \"label\": \"Mail.From CMD\",\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_to_cmd\",\n \"label\": \"Mail.To CMD\",\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_from\",\n \"label\": \"Mail.From\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"email\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_to\",\n \"label\": \"Mail.To\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"email\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_cc\",\n \"label\": \"Mail.CC\",\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_bcc\",\n \"label\": \"Mail.BCC\",\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_subject\",\n \"label\": \"Mail.Subject\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"decode_of_base64\",\n \"param\": \"$.mail_subject_charset\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_subject_charset\",\n \"label\": \"Mail.Subject Charset\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_content\",\n \"label\": \"Mail.Content\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_content_charset\",\n \"label\": \"Mail.Content Charset\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_attachment_name\",\n \"label\": \"Mail.Attachment\",\n \"doc\": {\n \"format\": {\n \"functions\": \"decode_of_base64\",\n \"param\": \"$.mail_attachment_name_charset\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_attachment_name_charset\",\n \"label\": \"Mail.Attachment Charset\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_attachment_content\",\n \"label\": \"Mail.Attachment Content\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_eml_file\",\n \"label\": \"Mail.EML File\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"file\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_snapshot\",\n \"label\": \"Mail.Snapshot\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"dns_message_id\",\n \"label\": \"DNS.Message ID\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_qr\",\n \"label\": \"DNS.QR\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"QUERY\"\n },\n {\n \"code\": \"1\",\n \"value\": \"REESPONSE\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_opcode\",\n \"label\": \"DNS.OPCODE\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"QUERY\"\n },\n {\n \"code\": \"1\",\n \"value\": \"IQUERY\"\n },\n {\n \"code\": \"2\",\n \"value\": \"STATUS\"\n },\n {\n \"code\": \"5\",\n \"value\": \"UPDATE\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_aa\",\n \"label\": \"DNS.AA\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_tc\",\n \"label\": \"DNS.TC\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_rd\",\n \"label\": \"DNS.RD\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_ra\",\n \"label\": \"DNS.RA\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_rcode\",\n \"label\": \"DNS.RCODE\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_qdcount\",\n \"label\": \"DNS.QDCOUNT\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_ancount\",\n \"label\": \"DNS.ANCOUNT\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_nscount\",\n \"label\": \"DNS.NSCOUNT\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_arcount\",\n \"label\": \"DNS.ARCOUNT\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_qname\",\n \"label\": \"DNS.QNAME\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"dns_qtype\",\n \"label\": \"DNS.QTYPE\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"A\"\n },\n {\n \"code\": \"2\",\n \"value\": \"NS\"\n },\n {\n \"code\": \"5\",\n \"value\": \"CNAME\"\n },\n {\n \"code\": \"6\",\n \"value\": \"SOA\"\n },\n {\n \"code\": \"11\",\n \"value\": \"WKS\"\n },\n {\n \"code\": \"12\",\n \"value\": \"PTR\"\n },\n {\n \"code\": \"13\",\n \"value\": \"HINFO\"\n },\n {\n \"code\": \"11\",\n \"value\": \"WKS\"\n },\n {\n \"code\": \"15\",\n \"value\": \"MX\"\n },\n {\n \"code\": \"28\",\n \"value\": \"AAAA\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_qclass\",\n \"label\": \"DNS.QCLASS\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_cname\",\n \"label\": \"DNS.CNAME\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"dns_sub\",\n \"label\": \"DNS.SUB\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"DNS\"\n },\n {\n \"code\": \"2\",\n \"value\": \"DNSSEC\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_rr\",\n \"label\": \"DNS.RR\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_version\",\n \"label\": \"SSL.Version\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_sni\",\n \"label\": \"SSL.SNI\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"sub_domain\",\n \"appendTo\": \"http_domain\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_san\",\n \"label\": \"SSL.SAN\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_cn\",\n \"label\": \"SSL.CN\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_pinningst\",\n \"label\": \"SSL.Pinning\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"Not Pinning\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Pinning\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Maybe Pinning\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_intercept_state\",\n \"label\": \"SSL.Intercept State\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"Passthrough\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Intercept\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Shutdown\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_server_side_latency\",\n \"label\": \"SSL.Server Side Latency(ms)\",\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_client_side_latency\",\n \"label\": \"SSL.Client Side Latency(ms)\",\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_server_side_version\",\n \"label\": \"SSL.Server Side Version\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_client_side_version\",\n \"label\": \"SSL.Client Side Version\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_cert_verify\",\n \"label\": \"SSL.Certificate Verify\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"No\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Yes\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_error\",\n \"label\": \"SSL.Error\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_con_latency_ms\",\n \"label\": \"SSL.Connection Latency(ms)\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_ja3_fingerprint\",\n \"label\": \"SSL.JA3\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_ja3_hash\",\n \"label\": \"SSL.JA3 hash\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_cert_issuer\",\n \"label\": \"SSL.Issuer\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_cert_subject\",\n \"label\": \"SSL.Subject\",\n \"type\": \"string\"\n },\n {\n \"name\": \"quic_version\",\n \"label\": \"QUIC.Version\",\n \"type\": \"string\"\n },\n {\n \"name\": \"quic_sni\",\n \"label\": \"QUIC.SNI\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"sub_domain\",\n \"appendTo\": \"http_domain\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"quic_user_agent\",\n \"label\": \"QUIC.User Agent\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ftp_account\",\n \"label\": \"FTP.Account\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ftp_url\",\n \"label\": \"FTP.URL\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ftp_content\",\n \"label\": \"FTP.Content\",\n \"type\": \"string\"\n },\n {\n \"name\": \"bgp_type\",\n \"label\": \"BGP.Type\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"bgp_as_num\",\n \"label\": \"BGP.AS Number\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"bgp_route\",\n \"label\": \"BGP.Route\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"voip_calling_account\",\n \"label\": \"VoIP.Calling Account\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"voip_called_account\",\n \"label\": \"VoIP.Called Account\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"voip_calling_number\",\n \"label\": \"VoIP.Calling Number\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"voip_called_number\",\n \"label\": \"VoIP.Called Number\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"streaming_media_url\",\n \"label\": \"Streaming.Media URL\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"streaming_media_protocol\",\n \"label\": \"Streaming.Media Protocol\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"app_extra_info\",\n \"label\": \"APP.Extra Info\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_call_id\",\n \"label\": \"SIP.Call-ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_originator_description\",\n \"label\": \"SIP.Originator\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_responder_description\",\n \"label\": \"SIP.Responder\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_user_agent\",\n \"label\": \"SIP.User-Agent\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_server\",\n \"label\": \"SIP.Server\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_originator_sdp_connect_ip\",\n \"label\": \"SIP.Originator IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_originator_sdp_media_port\",\n \"label\": \"SIP.Originator Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"sip_originator_sdp_media_type\",\n \"label\": \"SIP.Originator Media Type\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_originator_sdp_content\",\n \"label\": \"SIP.Originator Content\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_responder_sdp_connect_ip\",\n \"label\": \"SIP.Responder IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_responder_sdp_media_port\",\n \"label\": \"SIP.Responder Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"sip_responder_sdp_media_type\",\n \"label\": \"SIP.Responder Media Type\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_responder_sdp_content\",\n \"label\": \"SIP.Responder Content\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_duration\",\n \"label\": \"SIP.Duration\",\n \"type\": \"int\"\n },\n {\n \"name\": \"sip_bye\",\n \"label\": \"SIP.Bye\",\n \"type\": \"string\"\n },\n {\n \"name\": \"rtp_payload_type_c2s\",\n \"label\": \"RTP.Payload Type(c2s)\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"PCMU\"\n },\n {\n \"code\": \"1\",\n \"value\": \"1016\"\n },\n {\n \"code\": \"2\",\n \"value\": \"G721\"\n },\n {\n \"code\": \"3\",\n \"value\": \"GSM\"\n },\n {\n \"code\": \"4\",\n \"value\": \"G723\"\n },\n {\n \"code\": \"5\",\n \"value\": \"DVI4_8000\"\n },\n {\n \"code\": \"6\",\n \"value\": \"DVI4_16000\"\n },\n {\n \"code\": \"7\",\n \"value\": \"LPC\"\n },\n {\n \"code\": \"8\",\n \"value\": \"PCMA\"\n },\n {\n \"code\": \"9\",\n \"value\": \"G722\"\n },\n {\n \"code\": \"10\",\n \"value\": \"L16_STEREO\"\n },\n {\n \"code\": \"11\",\n \"value\": \"L16_MONO\"\n },\n {\n \"code\": \"12\",\n \"value\": \"QCELP\"\n },\n {\n \"code\": \"13\",\n \"value\": \"CN\"\n },\n {\n \"code\": \"14\",\n \"value\": \"MPA\"\n },\n {\n \"code\": \"15\",\n \"value\": \"G728\"\n },\n {\n \"code\": \"16\",\n \"value\": \"DVI4_11025\"\n },\n {\n \"code\": \"17\",\n \"value\": \"DVI4_22050\"\n },\n {\n \"code\": \"18\",\n \"value\": \"G729\"\n },\n {\n \"code\": \"19\",\n \"value\": \"CN_OLD\"\n },\n {\n \"code\": \"25\",\n \"value\": \"CELB\"\n },\n {\n \"code\": \"26\",\n \"value\": \"JPEG\"\n },\n {\n \"code\": \"28\",\n \"value\": \"NV\"\n },\n {\n \"code\": \"31\",\n \"value\": \"H261\"\n },\n {\n \"code\": \"32\",\n \"value\": \"MPV\"\n },\n {\n \"code\": \"33\",\n \"value\": \"MP2T\"\n },\n {\n \"code\": \"34\",\n \"value\": \"H263\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"rtp_payload_type_s2c\",\n \"label\": \"RTP.Payload Type(s2c)\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"PCMU\"\n },\n {\n \"code\": \"1\",\n \"value\": \"1016\"\n },\n {\n \"code\": \"2\",\n \"value\": \"G721\"\n },\n {\n \"code\": \"3\",\n \"value\": \"GSM\"\n },\n {\n \"code\": \"4\",\n \"value\": \"G723\"\n },\n {\n \"code\": \"5\",\n \"value\": \"DVI4_8000\"\n },\n {\n \"code\": \"6\",\n \"value\": \"DVI4_16000\"\n },\n {\n \"code\": \"7\",\n \"value\": \"LPC\"\n },\n {\n \"code\": \"8\",\n \"value\": \"PCMA\"\n },\n {\n \"code\": \"9\",\n \"value\": \"G722\"\n },\n {\n \"code\": \"10\",\n \"value\": \"L16_STEREO\"\n },\n {\n \"code\": \"11\",\n \"value\": \"L16_MONO\"\n },\n {\n \"code\": \"12\",\n \"value\": \"QCELP\"\n },\n {\n \"code\": \"13\",\n \"value\": \"CN\"\n },\n {\n \"code\": \"14\",\n \"value\": \"MPA\"\n },\n {\n \"code\": \"15\",\n \"value\": \"G728\"\n },\n {\n \"code\": \"16\",\n \"value\": \"DVI4_11025\"\n },\n {\n \"code\": \"17\",\n \"value\": \"DVI4_22050\"\n },\n {\n \"code\": \"18\",\n \"value\": \"G729\"\n },\n {\n \"code\": \"19\",\n \"value\": \"CN_OLD\"\n },\n {\n \"code\": \"25\",\n \"value\": \"CELB\"\n },\n {\n \"code\": \"26\",\n \"value\": \"JPEG\"\n },\n {\n \"code\": \"28\",\n \"value\": \"NV\"\n },\n {\n \"code\": \"31\",\n \"value\": \"H261\"\n },\n {\n \"code\": \"32\",\n \"value\": \"MPV\"\n },\n {\n \"code\": \"33\",\n \"value\": \"MP2T\"\n },\n {\n \"code\": \"34\",\n \"value\": \"H263\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"rtp_pcap_path\",\n \"label\": \"RTP.PCAP\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"files\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"rtp_originator_dir\",\n \"label\": \"RTP.Direction\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"unknown\"\n },\n {\n \"code\": \"1\",\n \"value\": \"c2s\"\n },\n {\n \"code\": \"2\",\n \"value\": \"s2c\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n }\n ]\n}', '0ad2746c6d311e1b7c15ac38d931b869', '2021-06-01 21:56:36', '2021-06-01 21:56:36', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (14, 'disks_cluster.json', 'Galaxy', '{\n \"namespace\": \"system\",\n \"type\": \"record\",\n \"name\": \"disks_cluster\",\n \"fields\": [\n {\n \"name\": \"name\",\n \"type\": \"string\"\n }\n ]\n}', 'c8840a8305bf77b4279f1a13465f70bf', '2021-06-01 21:56:39', '2021-06-01 21:56:39', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (15, 'druid-filter.json', 'Galaxy', '{\n \"version\": \"1.0\",\n \"name\": \"connection_record_log\",\n \"namespace\": \"druid\",\n \"filters\": [\n {\n \"name\":\"@start\",\n \"value\": \"\'2021-01-11 10:00:00\'\"\n },\n {\n \"name\":\"@end\",\n \"value\": \"\'2021-01-13 11:00:00\'\"\n },\n {\n \"name\":\"@common_filter\",\n \"value\": [\n \"common_client_ip=\'192.168.44.21\'and common_server_port=443\"\n ]\n }\n ]\n}', '6cd73cef23cc0cbdac044d039a3c4d36', '2021-06-01 21:56:40', '2021-06-01 21:56:40', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (16, 'engine-filter.json', 'Galaxy', '{\n \"version\": \"1.0\",\n \"name\": \"connection_record_log\",\n \"namespace\": \"Engine\",\n \"filters\": [\n {\n \"name\":\"@start\",\n \"value\": \"\'2021-01-11 10:00:00\'\"\n },\n {\n \"name\":\"@end\",\n \"value\": \"\'2021-01-13 11:00:00\'\"\n },\n {\n \"name\":\"@common_filter\",\n \"value\": [\n \"common_client_ip=\'192.168.44.21\'and common_server_port=443\",\n \"common_server_ip=\'8.8.8.8\' and common_server_port=443\",\n \"http_domain like \'%baidu%\' and common_server_port=443\",\n \"http_domain like \'%baidu%\'and common_server_port=443\"\n ]\n }\n ]\n}', 'd4ad147933a8288c5dcf0763bd8938c9', '2021-06-01 21:56:41', '2021-06-01 21:56:41', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (17, 'gtpc_record_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"gtpc_record_log\",\n \"namespace\": \"tsg_galaxy_v3\",\n \"doc\": {\n \"primary_key\": \"common_log_id\",\n \"partition_key\": \"common_recv_time\",\n \"functions\": {\n \"aggregation\": [\n {\n \"name\": \"COUNT\",\n \"label\": \"COUNT\",\n \"function\": \"count(expr)\"\n },\n {\n \"name\": \"COUNT_DISTINCT\",\n \"label\": \"COUNT_DISTINCT\",\n \"function\": \"count(distinct expr)\"\n },\n {\n \"name\": \"AVG\",\n \"label\": \"AVG\",\n \"function\": \"avg(expr)\"\n },\n {\n \"name\": \"SUM\",\n \"label\": \"SUM\",\n \"function\": \"sum(expr)\"\n },\n {\n \"name\": \"MAX\",\n \"label\": \"MAX\",\n \"function\": \"max(expr)\"\n },\n {\n \"name\": \"MIN\",\n \"label\": \"MIN\",\n \"function\": \"min(expr)\"\n }\n ],\n \"operator\": [\n {\n \"name\": \"=\",\n \"label\": \"=\",\n \"function\": \"expr = value\"\n },\n {\n \"name\": \"!=\",\n \"label\": \"!=\",\n \"function\": \"expr != value\"\n },\n {\n \"name\": \">\",\n \"label\": \">\",\n \"function\": \"expr > value\"\n },\n {\n \"name\": \"<\",\n \"label\": \"<\",\n \"function\": \"expr < value\"\n },\n {\n \"name\": \">=\",\n \"label\": \">=\",\n \"function\": \"expr >= value\"\n },\n {\n \"name\": \"<=\",\n \"label\": \"<=\",\n \"function\": \"expr <= value\"\n },\n {\n \"name\": \"in\",\n \"label\": \"IN\",\n \"function\": \"expr in (values)\"\n },\n {\n \"name\": \"not in\",\n \"label\": \"NOT IN\",\n \"function\": \"expr not in (values)\"\n },\n {\n \"name\": \"like\",\n \"label\": \"LIKE\",\n \"function\": \"expr like value\"\n },\n {\n \"name\": \"not like\",\n \"label\": \"NOT LIKE\",\n \"function\": \"expr not like value\"\n },\n {\n \"name\": \"notEmpty\",\n \"label\": \"NOT EMPTY\",\n \"function\": \"notEmpty(expr)\"\n },\n {\n \"name\": \"empty\",\n \"label\": \"EMPTY\",\n \"function\": \"empty(expr)\"\n }\n ]\n },\n \"schema_query\": {\n \"dimensions\": [\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_client_location\",\n \"common_server_location\",\n \"common_client_port\",\n \"common_server_port\",\n \"common_schema_type\",\n \"common_l4_protocol\",\n \"common_l7_protocol\",\n \"common_data_center\",\n \"common_client_asn\",\n \"common_server_asn\",\n \"common_start_time\",\n \"common_end_time\",\n \"gtp_version\",\n \"gtp_apn\",\n \"gtp_imei\",\n \"gtp_imsi\",\n \"gtp_phone_number\"\n ],\n \"metrics\": [\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_sessions\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"gtp_version\",\n \"gtp_apn\",\n \"gtp_imei\",\n \"gtp_imsi\",\n \"gtp_phone_number\"\n ],\n \"filters\": [\n \"common_address_type\",\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_client_port\",\n \"common_server_port\",\n \"common_client_location\",\n \"common_server_location\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_l4_protocol\",\n \"common_l7_protocol\",\n \"common_stream_dir\",\n \"common_direction\",\n \"common_data_center\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_schema_type\",\n \"common_client_asn\",\n \"common_server_asn\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"gtp_version\",\n \"gtp_apn\",\n \"gtp_imei\",\n \"gtp_imsi\",\n \"gtp_phone_number\",\n \"gtp_end_user_ipv4\",\n \"gtp_end_user_ipv6\",\n \"gtp_uplink_teid\",\n \"gtp_downlink_teid\",\n \"gtp_msg_type\"\n ],\n \"references\": {\n \"aggregation\": [\n {\n \"type\": \"int\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"long\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"float\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"double\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"string\",\n \"functions\": \"COUNT,COUNT_DISTINCT\"\n },\n {\n \"type\": \"date\",\n \"functions\": \"COUNT,COUNT_DISTINCT,MAX,MIN\"\n },\n {\n \"type\": \"timestamp\",\n \"functions\": \"COUNT,COUNT_DISTINCT,MAX,MIN\"\n }\n ],\n \"operator\": [\n {\n \"type\": \"int\",\n \"functions\": \"=,!=,>,<,>=,<=,in,not in\"\n },\n {\n \"type\": \"long\",\n \"functions\": \"=,!=,>,<,>=,<=,in,not in\"\n },\n {\n \"type\": \"float\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"double\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"string\",\n \"functions\": \"=,!=,in,not in,like,not like,notEmpty,empty\"\n },\n {\n \"type\": \"date\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"timestamp\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n }\n ]\n },\n \"details\": {\n \"general\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_stream_trace_id\",\n \"common_direction\",\n \"common_stream_dir\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_processing_time\",\n \"common_entrance_id\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_data_center\",\n \"common_sled_ip\"\n ],\n \"action\": [\n \"common_action\",\n \"common_sub_action\",\n \"common_policy_id\",\n \"common_user_tags\",\n \"common_user_region\"\n ],\n \"source\": [\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_client_port\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\"\n ],\n \"destination\": [\n \"common_server_ip\",\n \"common_external_ip\",\n \"common_server_port\",\n \"common_server_location\",\n \"common_server_asn\"\n ],\n \"application\": [\n \"common_app_id\",\n \"common_app_label\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_protocol_label\",\n \"common_service_category\",\n \"common_service\",\n \"common_l4_protocol\"\n ],\n \"transmission\": [\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_first_ttl\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\"\n ],\n \"other\": [\n \"common_address_type\",\n \"common_schema_type\",\n \"common_device_tag\",\n \"common_encapsulation\",\n \"common_tunnels\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\"\n ]\n }\n },\n \"schema_type\": {\n \"GTP-C\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_service_category\",\n \"common_l7_protocol\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"gtp_version\",\n \"gtp_apn\",\n \"gtp_imei\",\n \"gtp_imsi\",\n \"gtp_phone_number\",\n \"gtp_end_user_ipv4\",\n \"gtp_end_user_ipv6\",\n \"gtp_uplink_teid\",\n \"gtp_downlink_teid\",\n \"gtp_msg_type\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"gtp_version\",\n \"gtp_msg_type\",\n \"gtp_imsi\",\n \"gtp_imei\",\n \"gtp_phone_number\",\n \"common_client_ip\",\n \"common_server_ip\"\n ]\n }\n },\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"gtp_version\",\n \"gtp_msg_type\",\n \"gtp_imsi\",\n \"gtp_imei\",\n \"gtp_phone_number\",\n \"common_client_ip\",\n \"common_server_ip\"\n ],\n \"tunnel_type\": {\n \"GTP\": [\n {\n \"name\": \"gtp_sgw_ip\",\n \"label\": \"S-GW IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_pgw_ip\",\n \"label\": \"P-GW IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_sgw_port\",\n \"label\": \"S-GW Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"gtp_pgw_port\",\n \"label\": \"P-GW Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"gtp_uplink_teid\",\n \"label\": \"Uplink TEID\",\n \"type\": \"long\"\n },\n {\n \"name\": \"gtp_downlink_teid\",\n \"label\": \"Downlink TEID\",\n \"type\": \"long\"\n }\n ],\n \"MPLS\": [\n {\n \"name\": \"mpls_c2s_direction_label\",\n \"label\": \"Multiprotocol Label(c2s)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n },\n {\n \"name\": \"mpls_s2c_direction_label\",\n \"label\": \"Multiprotocol Label(s2c)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n }\n ],\n \"VLAN\": [\n {\n \"name\": \"vlan_c2s_direction_id\",\n \"label\": \"VLAN Direction(c2s)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n },\n {\n \"name\": \"vlan_s2c_direction_id\",\n \"label\": \"VLAN Direction(s2c)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n }\n ],\n \"ETHERNET\": [\n {\n \"name\": \"source_mac\",\n \"label\": \"Source MAC\",\n \"type\": \"string\"\n },\n {\n \"name\": \"destination_mac\",\n \"label\": \"Destination MAC\",\n \"type\": \"string\"\n }\n ],\n \"MULTIPATH_ETHERNET\": [\n {\n \"name\": \"c2s_source_mac\",\n \"label\": \"Source MAC(c2s)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"c2s_destination_mac\",\n \"label\": \"Destination MAC(c2s)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_source_mac\",\n \"label\": \"Source MAC(s2c)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_destination_mac\",\n \"label\": \"Destination MAC(s2c)\",\n \"type\": \"string\"\n }\n ]\n }\n },\n \"fields\": [\n {\n \"name\": \"common_recv_time\",\n \"label\": \"Receive Time\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_log_id\",\n \"label\": \"Log ID\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"snowflake_id\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_policy_id\",\n \"label\": \"Policy ID\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_subscriber_id\",\n \"label\": \"Subscriber ID\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_imei\",\n \"label\": \"IMEI\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_imsi\",\n \"label\": \"IMSI\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_phone_number\",\n \"label\": \"Phone Number\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_ip\",\n \"label\": \"Client IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"geo_asn\",\n \"appendTo\": \"common_client_asn\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_internal_ip\",\n \"label\": \"Internal IP\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"if\",\n \"param\": \"$.common_direction=69,$.common_client_ip,$.common_server_ip\"\n },\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_port\",\n \"label\": \"Client Port\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_l4_protocol\",\n \"label\": \"L4 Protocol\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_address_type\",\n \"label\": \"Address Type\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"4\",\n \"value\": \"ipv4\"\n },\n {\n \"code\": \"6\",\n \"value\": \"ipv6\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_server_ip\",\n \"label\": \"Server IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"geo_asn\",\n \"appendTo\": \"common_server_asn\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_port\",\n \"label\": \"Server Port\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_external_ip\",\n \"label\": \"External IP\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"if\",\n \"param\": \"$.common_direction=73,$.common_client_ip,$.common_server_ip\"\n },\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_action\",\n \"label\": \"Action\",\n \"doc\": {\n \"visibility\": \"hidden\",\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"None\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Intercept\"\n },\n {\n \"code\": \"16\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"128\",\n \"value\": \"Allow\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_direction\",\n \"label\": \"Direction\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"69\",\n \"value\": \"outbound\"\n },\n {\n \"code\": \"73\",\n \"value\": \"inbound\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_entrance_id\",\n \"label\": \"Entrance ID\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_sled_ip\",\n \"label\": \"Sled IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_location\",\n \"label\": \"Client Location\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_asn\",\n \"label\": \"Client ASN\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_location\",\n \"label\": \"Server Location\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_asn\",\n \"label\": \"Server ASN\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sessions\",\n \"label\": \"Sessions\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_num\",\n \"label\": \"Packets Sent\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_num\",\n \"label\": \"Packets Received\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_num\",\n \"label\": \"Bytes Sent\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_num\",\n \"label\": \"Bytes Received\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_diff\",\n \"label\": \"Packets Sent(Diff)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_diff\",\n \"label\": \"Packets Received(Diff)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_diff\",\n \"label\": \"Bytes Sent(Diff)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_diff\",\n \"label\": \"Bytes Received(Diff)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_service\",\n \"label\": \"Service\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_schema_type\",\n \"label\": \"Schema Type\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"GTP-C\",\n \"value\": \"GTP-C\"\n }\n ]\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_user_tags\",\n \"label\": \"User Tags\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sub_action\",\n \"label\": \"Sub Action\",\n \"doc\": {\n \"data\": [\n {\n \"code\": \"allow\",\n \"value\": \"Allow\"\n },\n {\n \"code\": \"deny\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"monitor\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"replace\",\n \"value\": \"Replace\"\n },\n {\n \"code\": \"redirect\",\n \"value\": \"Redirect\"\n },\n {\n \"code\": \"insert\",\n \"value\": \"Insert\"\n },\n {\n \"code\": \"hijack\",\n \"value\": \"Hijack\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_user_region\",\n \"label\": \"User Region\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_device_id\",\n \"label\": \"Device ID\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_link_id\",\n \"label\": \"Link ID\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_isp\",\n \"label\": \"ISP\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_device_tag\",\n \"label\": \"Device Tag\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_data_center\",\n \"label\": \"Data Center\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"City A\",\n \"value\": \"City A\"\n },\n {\n \"code\": \"City B\",\n \"value\": \"City B\"\n },\n {\n \"code\": \"City C\",\n \"value\": \"City C\"\n },\n {\n \"code\": \"City D\",\n \"value\": \"City D\"\n },\n {\n \"code\": \"City E\",\n \"value\": \"City E\"\n },\n {\n \"code\": \"City F\",\n \"value\": \"City F\"\n },\n {\n \"code\": \"City G\",\n \"value\": \"City G\"\n },\n {\n \"code\": \"City H\",\n \"value\": \"City H\"\n },\n {\n \"code\": \"City I\",\n \"value\": \"City I\"\n },\n {\n \"code\": \"City J\",\n \"value\": \"City J\"\n },\n {\n \"code\": \"City K\",\n \"value\": \"City K\"\n },\n {\n \"code\": \"City L\",\n \"value\": \"City L\"\n },\n {\n \"code\": \"City M\",\n \"value\": \"City M\"\n },\n {\n \"code\": \"City N\",\n \"value\": \"City N\"\n },\n {\n \"code\": \"City O\",\n \"value\": \"City O\"\n },\n {\n \"code\": \"City P\",\n \"value\": \"City P\"\n },\n {\n \"code\": \"City Q\",\n \"value\": \"City Q\"\n },\n {\n \"code\": \"City R\",\n \"value\": \"City R\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_encapsulation\",\n \"label\": \"Encapsulation\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"Ethernet\"\n },\n {\n \"code\": \"8\",\n \"value\": \"PPP\"\n },\n {\n \"code\": \"12\",\n \"value\": \"CiscoHDLC\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_app_label\",\n \"label\": \"Application Label\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_tunnels\",\n \"label\": \"Tunnels\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_protocol_label\",\n \"label\": \"Protocol Label\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_app_id\",\n \"label\": \"Application ID\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_app_surrogate_id\",\n \"label\": \"Surrogate ID\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_l7_protocol\",\n \"label\": \"L7 Protocol\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_service_category\",\n \"label\": \"FQDN Category\",\n \"doc\": {\n \"visibility\": \"disabled\",\n \"dict_location\": {\n \"path\": \"/v1/category/dict\",\n \"key\": \"categoryId\",\n \"value\": \"categoryName\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_start_time\",\n \"label\": \"Start Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_end_time\",\n \"label\": \"End Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"get_value\",\n \"appendTo\": \"common_recv_time\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_establish_latency_ms\",\n \"label\": \"Establish Latency(ms)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_con_duration_ms\",\n \"label\": \"Duration(ms)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_stream_dir\",\n \"label\": \"Stream Direction\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"c2s\"\n },\n {\n \"code\": \"2\",\n \"value\": \"s2c\"\n },\n {\n \"code\": \"3\",\n \"value\": \"double\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_address_list\",\n \"label\": \"Address List\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_has_dup_traffic\",\n \"label\": \"Duplication Traffic\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"No\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Yes\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_stream_error\",\n \"label\": \"Stream Error\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_stream_trace_id\",\n \"label\": \"Session ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_link_info_c2s\",\n \"label\": \"Link Info(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_link_info_s2c\",\n \"label\": \"Link Info(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_c2s_ipfrag_num\",\n \"label\": \"Fragmentation Packets(c2s)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_ipfrag_num\",\n \"label\": \"Fragmentation Packets(s2c)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_tcp_lostlen\",\n \"label\": \"Sequence Gap Loss(c2s)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_tcp_lostlen\",\n \"label\": \"Sequence Gap Loss(s2c)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_tcp_unorder_num\",\n \"label\": \"Unorder Packets(c2s)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_tcp_unorder_num\",\n \"label\": \"Unorder Packets(s2c)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_retrans\",\n \"label\": \"Packet Retransmission(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_retrans\",\n \"label\": \"Packet Retransmission(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_retrans\",\n \"label\": \"Byte Retransmission(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_retrans\",\n \"label\": \"Byte Retransmission(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_tcp_client_isn\",\n \"label\": \"TCP Client ISN\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_tcp_server_isn\",\n \"label\": \"TCP Server ISN\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_first_ttl\",\n \"label\": \"First TTL\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_processing_time\",\n \"label\": \"Processing Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"current_timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"gtp_version\",\n \"label\": \"Version\",\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_apn\",\n \"label\": \"APN\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_imei\",\n \"label\": \"IMEI\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_imsi\",\n \"label\": \"IMSI\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_phone_number\",\n \"label\": \"Phone Number\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_uplink_teid\",\n \"label\": \"Uplink TEID\",\n \"type\": \"long\"\n },\n {\n \"name\": \"gtp_downlink_teid\",\n \"label\": \"Downlink TEID\",\n \"type\": \"long\"\n },\n {\n \"name\": \"gtp_msg_type\",\n \"label\": \"Message Type\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"create\",\n \"value\": \"create\"\n },\n {\n \"code\": \"modify\",\n \"value\": \"modify\"\n },\n {\n \"code\": \"delete\",\n \"value\": \"delete\"\n }\n ]\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_end_user_ipv4\",\n \"label\": \"End User Address V4\",\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_end_user_ipv6\",\n \"label\": \"End User Address V6\",\n \"type\": \"string\"\n }\n ]\n}', '38b0bb16bdccb7667b5908b597f9dd0b', '2021-06-01 21:56:41', '2021-06-01 21:56:41', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (18, 'liveChart_history.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"liveChart_history\",\n \"task\": \"Protocol-Distribution\",\n \"in\": \"CONNECTION-RECORD-LOG\",\n \"out\": \"TRAFFIC-PROTOCOL-STAT-LOG\",\n \"doc\": {\n \"timestamp\": {\n \"name\": \"stat_time\",\n \"type\": \"Long\"\n },\n \"dimensions\": [\n {\n \"name\": \"protocol_id\",\n \"fieldName\": \"common_protocol_label\",\n \"type\": \"String\"\n },\n {\n \"name\": \"entrance_id\",\n \"fieldName\": \"common_entrance_id\",\n \"type\": \"String\"\n },\n {\n \"name\": \"isp\",\n \"fieldName\": \"common_isp\",\n \"type\": \"String\"\n },\n {\n \"name\": \"data_center\",\n \"fieldName\": \"common_data_center\",\n \"type\": \"String\"\n }\n ],\n \"metrics\": [\n {\n \"function\": \"sum\",\n \"name\": \"sessions\",\n \"fieldName\": \"common_sessions\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"c2s_byte_num\",\n \"fieldName\": \"common_c2s_byte_diff\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"s2c_byte_num\",\n \"fieldName\": \"common_s2c_byte_diff\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"c2s_pkt_num\",\n \"fieldName\": \"common_c2s_pkt_diff\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"s2c_pkt_num\",\n \"fieldName\": \"common_s2c_pkt_diff\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"c2s_ipfrag_num\",\n \"fieldName\": \"common_c2s_ipfrag_num\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"s2c_ipfrag_num\",\n \"fieldName\": \"common_s2c_ipfrag_num\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"c2s_tcp_lostlen\",\n \"fieldName\": \"common_c2s_tcp_lostlen\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"s2c_tcp_lostlen\",\n \"fieldName\": \"common_s2c_tcp_lostlen\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"c2s_tcp_unorder_num\",\n \"fieldName\": \"common_c2s_tcp_unorder_num\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"s2c_tcp_unorder_num\",\n \"fieldName\": \"common_s2c_tcp_unorder_num\"\n },\n {\n \"function\": \"disCount\",\n \"name\": \"unique_sip_num\",\n \"fieldName\": \"common_server_ip\"\n },\n {\n \"function\": \"disCount\",\n \"name\": \"unique_cip_num\",\n \"fieldName\": \"common_client_ip\"\n }\n ],\n \"filters\": {\n \"type\": \"and\",\n \"fields\": [\n {\n \"fieldName\": \"common_protocol_label\",\n \"type\": \"not\",\n \"values\": null\n }\n ]\n },\n \"transforms\": [\n {\n \"function\": \"combination\",\n \"name\": \"protocol_id\",\n \"fieldName\": \"common_protocol_label\",\n \"parameters\": \"common_l7_protocol,.\"\n },\n {\n \"function\": \"combination\",\n \"name\": \"protocol_id\",\n \"fieldName\": \"common_protocol_label\",\n \"parameters\": \"common_app_label,.\"\n },\n {\n \"function\": \"hierarchy\",\n \"name\": \"protocol_id\",\n \"fieldName\": \"common_protocol_label\",\n \"parameters\": \".\"\n }\n ],\n \"action\": [\n {\n \"label\": \"Default\",\n \"metrics\": \"sessions,c2s_byte_num,s2c_byte_num,c2s_pkt_num,s2c_pkt_num,c2s_ipfrag_num,s2c_ipfrag_num,c2s_tcp_lostlen,s2c_tcp_lostlen,c2s_tcp_unorder_num,s2c_tcp_unorder_num\"\n }\n ],\n \"granularity\": {\n \"type\": \"period\",\n \"period\": \"5M\"\n }\n },\n \"fields\": []\n}', 'f356493e4ded7518b900256970b27b9b', '2021-06-01 21:56:42', '2021-06-01 21:56:42', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (19, 'liveChart_live.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"liveChart_live\",\n \"in\": \"LIVE-SESSION-RECORD-LOG\",\n \"out\": \"TRAFFIC-PROTOCOL-STAT-LOG\",\n \"task\": \"Protocol-Distribution\",\n \"doc\": {\n \"timestamp\": {\n \"name\": \"stat_time\",\n \"type\": \"Long\"\n },\n \"dimensions\": [\n {\n \"name\": \"protocol_id\",\n \"fieldName\": \"common_protocol_label\",\n \"type\": \"String\"\n },\n {\n \"name\": \"entrance_id\",\n \"fieldName\": \"common_entrance_id\",\n \"type\": \"String\"\n },\n {\n \"name\": \"isp\",\n \"fieldName\": \"common_isp\",\n \"type\": \"String\"\n },\n {\n \"name\": \"data_center\",\n \"fieldName\": \"common_data_center\",\n \"type\": \"String\"\n }\n ],\n \"metrics\": [\n {\n \"function\": \"sum\",\n \"name\": \"sessions\",\n \"fieldName\": \"common_sessions\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"c2s_byte_num\",\n \"fieldName\": \"common_c2s_byte_diff\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"s2c_byte_num\",\n \"fieldName\": \"common_s2c_byte_diff\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"c2s_pkt_num\",\n \"fieldName\": \"common_c2s_pkt_diff\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"s2c_pkt_num\",\n \"fieldName\": \"common_s2c_pkt_diff\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"c2s_ipfrag_num\",\n \"fieldName\": \"common_c2s_ipfrag_num\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"s2c_ipfrag_num\",\n \"fieldName\": \"common_s2c_ipfrag_num\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"c2s_tcp_lostlen\",\n \"fieldName\": \"common_c2s_tcp_lostlen\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"s2c_tcp_lostlen\",\n \"fieldName\": \"common_s2c_tcp_lostlen\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"c2s_tcp_unorder_num\",\n \"fieldName\": \"common_c2s_tcp_unorder_num\"\n },\n {\n \"function\": \"sum\",\n \"name\": \"s2c_tcp_unorder_num\",\n \"fieldName\": \"common_s2c_tcp_unorder_num\"\n },\n {\n \"function\": \"disCount\",\n \"name\": \"unique_sip_num\",\n \"fieldName\": \"common_server_ip\"\n },\n {\n \"function\": \"disCount\",\n \"name\": \"unique_cip_num\",\n \"fieldName\": \"common_client_ip\"\n }\n ],\n \"filters\": {\n \"type\": \"and\",\n \"fields\": [\n {\n \"fieldName\": \"common_protocol_label\",\n \"type\": \"not\",\n \"values\": null\n }\n ]\n },\n \"transforms\": [\n {\n \"function\": \"combination\",\n \"name\": \"protocol_id\",\n \"fieldName\": \"common_protocol_label\",\n \"parameters\": \"common_l7_protocol,.\"\n },\n {\n \"function\": \"combination\",\n \"name\": \"protocol_id\",\n \"fieldName\": \"common_protocol_label\",\n \"parameters\": \"common_app_label,.\"\n },\n {\n \"function\": \"hierarchy\",\n \"name\": \"protocol_id\",\n \"fieldName\": \"common_protocol_label\",\n \"parameters\": \".\"\n }\n ],\n \"action\": [\n {\n \"label\": \"Default\",\n \"metrics\": \"c2s_byte_num,s2c_byte_num,c2s_pkt_num,s2c_pkt_num\"\n }\n ],\n \"granularity\": {\n \"type\": \"period\",\n \"period\": \"5M\"\n }\n },\n \"fields\": []\n}', '1fedfdb46f033091e908316d585408e1', '2021-06-01 21:56:43', '2021-06-01 21:56:43', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (20, 'live_session_record_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"live_session_record_log\",\n \"namespace\": \"tsg_galaxy_v3\",\n \"doc\": {\n \"primary_key\": \"common_log_id\",\n \"partition_key\": \"common_recv_time\",\n \"functions\": {\n \"aggregation\": [\n {\n \"name\": \"COUNT\",\n \"label\": \"COUNT\",\n \"function\": \"count(expr)\"\n },\n {\n \"name\": \"COUNT_DISTINCT\",\n \"label\": \"COUNT_DISTINCT\",\n \"function\": \"count(distinct expr)\"\n },\n {\n \"name\": \"AVG\",\n \"label\": \"AVG\",\n \"function\": \"avg(expr)\"\n },\n {\n \"name\": \"SUM\",\n \"label\": \"SUM\",\n \"function\": \"sum(expr)\"\n },\n {\n \"name\": \"MAX\",\n \"label\": \"MAX\",\n \"function\": \"max(expr)\"\n },\n {\n \"name\": \"MIN\",\n \"label\": \"MIN\",\n \"function\": \"min(expr)\"\n }\n ],\n \"operator\": [\n {\n \"name\": \"=\",\n \"label\": \"=\",\n \"function\": \"expr = value\"\n },\n {\n \"name\": \"!=\",\n \"label\": \"!=\",\n \"function\": \"expr != value\"\n },\n {\n \"name\": \">\",\n \"label\": \">\",\n \"function\": \"expr > value\"\n },\n {\n \"name\": \"<\",\n \"label\": \"<\",\n \"function\": \"expr < value\"\n },\n {\n \"name\": \">=\",\n \"label\": \">=\",\n \"function\": \"expr >= value\"\n },\n {\n \"name\": \"<=\",\n \"label\": \"<=\",\n \"function\": \"expr <= value\"\n },\n {\n \"name\": \"in\",\n \"label\": \"IN\",\n \"function\": \"expr in (values)\"\n },\n {\n \"name\": \"not in\",\n \"label\": \"NOT IN\",\n \"function\": \"expr not in (values)\"\n },\n {\n \"name\": \"like\",\n \"label\": \"LIKE\",\n \"function\": \"expr like value\"\n },\n {\n \"name\": \"not like\",\n \"label\": \"NOT LIKE\",\n \"function\": \"expr not like value\"\n },\n {\n \"name\": \"notEmpty\",\n \"label\": \"NOT EMPTY\",\n \"function\": \"notEmpty(expr)\"\n },\n {\n \"name\": \"empty\",\n \"label\": \"EMPTY\",\n \"function\": \"empty(expr)\"\n }\n ]\n },\n \"schema_query\": {\n \"dimensions\": [\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_client_location\",\n \"common_server_location\",\n \"common_subscriber_id\",\n \"common_client_port\",\n \"common_server_port\",\n \"common_schema_type\",\n \"common_l4_protocol\",\n \"common_l7_protocol\",\n \"common_data_center\",\n \"common_client_asn\",\n \"common_server_asn\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"http_host\",\n \"http_domain\",\n \"http_url\",\n \"ssl_sni\",\n \"ssl_ja3_hash\"\n ],\n \"metrics\": [\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_subscriber_id\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_sessions\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"http_host\",\n \"http_domain\",\n \"http_url\",\n \"ssl_sni\",\n \"ssl_ja3_hash\"\n ],\n \"filters\": [\n \"common_address_type\",\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_client_port\",\n \"common_server_port\",\n \"common_client_location\",\n \"common_server_location\",\n \"common_subscriber_id\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_l4_protocol\",\n \"common_l7_protocol\",\n \"common_stream_dir\",\n \"common_direction\",\n \"common_data_center\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_schema_type\",\n \"common_client_asn\",\n \"common_server_asn\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"http_host\",\n \"http_domain\",\n \"http_url\",\n \"ssl_sni\",\n \"ssl_ja3_hash\"\n ],\n \"references\": {\n \"aggregation\": [\n {\n \"type\": \"int\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"long\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"float\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"double\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"string\",\n \"functions\": \"COUNT,COUNT_DISTINCT\"\n },\n {\n \"type\": \"date\",\n \"functions\": \"COUNT,COUNT_DISTINCT,MAX,MIN\"\n },\n {\n \"type\": \"timestamp\",\n \"functions\": \"COUNT,COUNT_DISTINCT,MAX,MIN\"\n }\n ],\n \"operator\": [\n {\n \"type\": \"int\",\n \"functions\": \"=,!=,>,<,>=,<=,in,not in\"\n },\n {\n \"type\": \"long\",\n \"functions\": \"=,!=,>,<,>=,<=,in,not in\"\n },\n {\n \"type\": \"float\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"double\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"string\",\n \"functions\": \"=,!=,in,not in,like,not like,notEmpty,empty\"\n },\n {\n \"type\": \"date\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"timestamp\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n }\n ]\n },\n \"details\": {\n \"general\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_stream_trace_id\",\n \"common_direction\",\n \"common_stream_dir\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_processing_time\",\n \"common_entrance_id\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_data_center\",\n \"common_sled_ip\"\n ],\n \"action\": [\n \"common_action\",\n \"common_sub_action\",\n \"common_policy_id\",\n \"common_user_tags\",\n \"common_user_region\"\n ],\n \"source\": [\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_client_port\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\"\n ],\n \"destination\": [\n \"common_server_ip\",\n \"common_external_ip\",\n \"common_server_port\",\n \"common_server_location\",\n \"common_server_asn\"\n ],\n \"application\": [\n \"common_app_id\",\n \"common_app_label\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_protocol_label\",\n \"common_service_category\",\n \"common_service\",\n \"common_l4_protocol\"\n ],\n \"transmission\": [\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_first_ttl\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\"\n ],\n \"other\": [\n \"common_address_type\",\n \"common_schema_type\",\n \"common_device_tag\",\n \"common_encapsulation\",\n \"common_tunnels\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\"\n ]\n }\n },\n \"schema_type\": {\n \"BASE\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_service_category\",\n \"common_l7_protocol\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"HTTP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_service_category\",\n \"common_l7_protocol\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"http_url\",\n \"http_host\",\n \"http_domain\",\n \"http_request_line\",\n \"http_response_line\",\n \"http_request_header\",\n \"http_response_header\",\n \"http_request_content\",\n \"http_response_content\",\n \"http_request_body\",\n \"http_response_body\",\n \"http_request_body_key\",\n \"http_response_body_key\",\n \"http_proxy_flag\",\n \"http_sequence\",\n \"http_snapshot\",\n \"http_cookie\",\n \"http_referer\",\n \"http_user_agent\",\n \"http_content_length\",\n \"http_content_type\",\n \"http_set_cookie\",\n \"http_version\",\n \"http_response_lantency_ms\",\n \"http_session_duration_ms\",\n \"http_action_file_size\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"http_url\",\n \"common_server_port\"\n ]\n },\n \"MAIL\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"mail_protocol_type\",\n \"mail_account\",\n \"mail_from_cmd\",\n \"mail_to_cmd\",\n \"mail_from\",\n \"mail_to\",\n \"mail_cc\",\n \"mail_bcc\",\n \"mail_subject\",\n \"mail_subject_charset\",\n \"mail_content\",\n \"mail_content_charset\",\n \"mail_attachment_name\",\n \"mail_attachment_name_charset\",\n \"mail_attachment_content\",\n \"mail_eml_file\",\n \"mail_snapshot\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"mail_from\",\n \"mail_to\",\n \"mail_subject\"\n ]\n },\n \"DNS\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"dns_message_id\",\n \"dns_qr\",\n \"dns_opcode\",\n \"dns_aa\",\n \"dns_tc\",\n \"dns_rd\",\n \"dns_ra\",\n \"dns_rcode\",\n \"dns_qdcount\",\n \"dns_ancount\",\n \"dns_nscount\",\n \"dns_arcount\",\n \"dns_qname\",\n \"dns_qtype\",\n \"dns_qclass\",\n \"dns_cname\",\n \"dns_sub\",\n \"dns_rr\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_client_ip\",\n \"dns_qr\",\n \"dns_qname\",\n \"dns_qtype\"\n ]\n },\n \"SSL\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"ssl_sni\",\n \"ssl_san\",\n \"ssl_cn\",\n \"ssl_pinningst\",\n \"ssl_intercept_state\",\n \"ssl_server_side_latency\",\n \"ssl_client_side_latency\",\n \"ssl_server_side_version\",\n \"ssl_client_side_version\",\n \"ssl_cert_verify\",\n \"ssl_error\",\n \"ssl_con_latency_ms\",\n \"ssl_ja3_fingerprint\",\n \"ssl_ja3_hash\",\n \"ssl_cert_issuer\",\n \"ssl_cert_subject\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"ssl_sni\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"QUIC\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"quic_version\",\n \"quic_sni\",\n \"quic_user_agent\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"quic_sni\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"FTP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"ftp_account\",\n \"ftp_url\",\n \"ftp_content\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"ftp_url\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"BGP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"bgp_type\",\n \"bgp_as_num\",\n \"bgp_route\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"bgp_type\",\n \"bgp_as_num\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"SIP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"sip_call_id\",\n \"sip_originator_description\",\n \"sip_responder_description\",\n \"sip_user_agent\",\n \"sip_server\",\n \"sip_originator_sdp_connect_ip\",\n \"sip_originator_sdp_media_port\",\n \"sip_originator_sdp_media_type\",\n \"sip_originator_sdp_content\",\n \"sip_responder_sdp_connect_ip\",\n \"sip_responder_sdp_media_port\",\n \"sip_responder_sdp_media_type\",\n \"sip_responder_sdp_content\",\n \"sip_duration\",\n \"sip_bye\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"sip_originator_description\",\n \"sip_responder_description\",\n \"sip_call_id\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"RTP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"rtp_payload_type_c2s\",\n \"rtp_payload_type_s2c\",\n \"rtp_pcap_path\",\n \"rtp_originator_dir\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\",\n \"rtp_pcap_path\",\n \"rtp_originator_dir\"\n ]\n },\n \"APP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"app_extra_info\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_app_id\",\n \"common_app_label\",\n \"app_extra_info\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n }\n },\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_schema_type\"\n ]\n },\n \"fields\": [\n {\n \"name\": \"common_recv_time\",\n \"label\": \"Receive Time\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_log_id\",\n \"label\": \"Log ID\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"snowflake_id\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_policy_id\",\n \"label\": \"Policy ID\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_subscriber_id\",\n \"label\": \"Subscriber ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_imei\",\n \"label\": \"IMEI\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_imsi\",\n \"label\": \"IMSI\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_phone_number\",\n \"label\": \"Phone Number\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_ip\",\n \"label\": \"Client IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"geo_asn,radius_match\",\n \"appendTo\": \"common_client_asn,common_subscriber_id\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_internal_ip\",\n \"label\": \"Internal IP\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"if\",\n \"param\": \"$.common_direction=69,$.common_client_ip,$.common_server_ip\"\n },\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_port\",\n \"label\": \"Client Port\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_l4_protocol\",\n \"label\": \"L4 Protocol\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_address_type\",\n \"label\": \"Address Type\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"4\",\n \"value\": \"ipv4\"\n },\n {\n \"code\": \"6\",\n \"value\": \"ipv6\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_server_ip\",\n \"label\": \"Server IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"geo_asn\",\n \"appendTo\": \"common_server_asn\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_port\",\n \"label\": \"Server Port\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_external_ip\",\n \"label\": \"External IP\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"if\",\n \"param\": \"$.common_direction=73,$.common_client_ip,$.common_server_ip\"\n },\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_action\",\n \"label\": \"Action\",\n \"doc\": {\n \"visibility\": \"hidden\",\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"None\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Intercept\"\n },\n {\n \"code\": \"16\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"128\",\n \"value\": \"Allow\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_direction\",\n \"label\": \"Direction\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"69\",\n \"value\": \"outbound\"\n },\n {\n \"code\": \"73\",\n \"value\": \"inbound\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_entrance_id\",\n \"label\": \"Entrance ID\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_sled_ip\",\n \"label\": \"Sled IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_location\",\n \"label\": \"Client Location\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_asn\",\n \"label\": \"Client ASN\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_location\",\n \"label\": \"Server Location\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_asn\",\n \"label\": \"Server ASN\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sessions\",\n \"label\": \"Sessions\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_num\",\n \"label\": \"Packets Sent\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_num\",\n \"label\": \"Packets Received\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_num\",\n \"label\": \"Bytes Sent\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_num\",\n \"label\": \"Bytes Received\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_diff\",\n \"label\": \"Packets Sent(Diff)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_diff\",\n \"label\": \"Packets Received(Diff)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_diff\",\n \"label\": \"Bytes Sent(Diff)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_diff\",\n \"label\": \"Bytes Received(Diff)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_service\",\n \"label\": \"Service\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_schema_type\",\n \"label\": \"Schema Type\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"BASE\",\n \"value\": \"BASE\"\n },\n {\n \"code\": \"MAIL\",\n \"value\": \"MAIL\"\n },\n {\n \"code\": \"DNS\",\n \"value\": \"DNS\"\n },\n {\n \"code\": \"HTTP\",\n \"value\": \"HTTP\"\n },\n {\n \"code\": \"SSL\",\n \"value\": \"SSL\"\n },\n {\n \"code\": \"FTP\",\n \"value\": \"FTP\"\n },\n {\n \"code\": \"SIP\",\n \"value\": \"SIP\"\n },\n {\n \"code\": \"RTP\",\n \"value\": \"RTP\"\n },\n {\n \"code\": \"APP\",\n \"value\": \"APP\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_user_tags\",\n \"label\": \"User Tags\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sub_action\",\n \"label\": \"Sub Action\",\n \"doc\": {\n \"data\": [\n {\n \"code\": \"allow\",\n \"value\": \"Allow\"\n },\n {\n \"code\": \"deny\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"monitor\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"replace\",\n \"value\": \"Replace\"\n },\n {\n \"code\": \"redirect\",\n \"value\": \"Redirect\"\n },\n {\n \"code\": \"insert\",\n \"value\": \"Insert\"\n },\n {\n \"code\": \"hijack\",\n \"value\": \"Hijack\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_user_region\",\n \"label\": \"User Region\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_device_id\",\n \"label\": \"Device ID\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_link_id\",\n \"label\": \"Link ID\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_isp\",\n \"label\": \"ISP\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_device_tag\",\n \"label\": \"Device Tag\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_data_center\",\n \"label\": \"Data Center\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"City A\",\n \"value\": \"City A\"\n },\n {\n \"code\": \"City B\",\n \"value\": \"City B\"\n },\n {\n \"code\": \"City C\",\n \"value\": \"City C\"\n },\n {\n \"code\": \"City D\",\n \"value\": \"City D\"\n },\n {\n \"code\": \"City E\",\n \"value\": \"City E\"\n },\n {\n \"code\": \"City F\",\n \"value\": \"City F\"\n },\n {\n \"code\": \"City G\",\n \"value\": \"City G\"\n },\n {\n \"code\": \"City H\",\n \"value\": \"City H\"\n },\n {\n \"code\": \"City I\",\n \"value\": \"City I\"\n },\n {\n \"code\": \"City J\",\n \"value\": \"City J\"\n },\n {\n \"code\": \"City K\",\n \"value\": \"City K\"\n },\n {\n \"code\": \"City L\",\n \"value\": \"City L\"\n },\n {\n \"code\": \"City M\",\n \"value\": \"City M\"\n },\n {\n \"code\": \"City N\",\n \"value\": \"City N\"\n },\n {\n \"code\": \"City O\",\n \"value\": \"City O\"\n },\n {\n \"code\": \"City P\",\n \"value\": \"City P\"\n },\n {\n \"code\": \"City Q\",\n \"value\": \"City Q\"\n },\n {\n \"code\": \"City R\",\n \"value\": \"City R\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_encapsulation\",\n \"label\": \"Encapsulation\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"Ethernet\"\n },\n {\n \"code\": \"8\",\n \"value\": \"PPP\"\n },\n {\n \"code\": \"12\",\n \"value\": \"CiscoHDLC\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_app_label\",\n \"label\": \"Application Label\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_tunnels\",\n \"label\": \"Tunnels\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_protocol_label\",\n \"label\": \"Protocol Label\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_app_id\",\n \"label\": \"Application ID\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_app_surrogate_id\",\n \"label\": \"Surrogate ID\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_l7_protocol\",\n \"label\": \"L7 Protocol\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_service_category\",\n \"label\": \"FQDN Category\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"dict_location\": {\n \"path\": \"/v1/category/dict\",\n \"key\": \"categoryId\",\n \"value\": \"categoryName\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_start_time\",\n \"label\": \"Start Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_end_time\",\n \"label\": \"End Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"get_value\",\n \"appendTo\": \"common_recv_time\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_establish_latency_ms\",\n \"label\": \"Establish Latency(ms)\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_con_duration_ms\",\n \"label\": \"Duration(ms)\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_stream_dir\",\n \"label\": \"Stream Direction\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"c2s\"\n },\n {\n \"code\": \"2\",\n \"value\": \"s2c\"\n },\n {\n \"code\": \"3\",\n \"value\": \"double\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_address_list\",\n \"label\": \"Address List\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_has_dup_traffic\",\n \"label\": \"Duplication Traffic\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"No\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Yes\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_stream_error\",\n \"label\": \"Stream Error\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_stream_trace_id\",\n \"label\": \"Session ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_link_info_c2s\",\n \"label\": \"Link Info(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_link_info_s2c\",\n \"label\": \"Link Info(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_c2s_ipfrag_num\",\n \"label\": \"Fragmentation Packets(c2s)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_ipfrag_num\",\n \"label\": \"Fragmentation Packets(s2c)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_tcp_lostlen\",\n \"label\": \"Sequence Gap Loss(c2s)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_tcp_lostlen\",\n \"label\": \"Sequence Gap Loss(s2c)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_tcp_unorder_num\",\n \"label\": \"Unorder Packets(c2s)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_tcp_unorder_num\",\n \"label\": \"Unorder Packets(s2c)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_retrans\",\n \"label\": \"Packet Retransmission(c2s)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_retrans\",\n \"label\": \"Packet Retransmission(s2c)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_retrans\",\n \"label\": \"Byte Retransmission(c2s)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_retrans\",\n \"label\": \"Byte Retransmission(s2c)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_tcp_client_isn\",\n \"label\": \"TCP Client ISN\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_tcp_server_isn\",\n \"label\": \"TCP Server ISN\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_first_ttl\",\n \"label\": \"First TTL\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_processing_time\",\n \"label\": \"Processing Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"current_timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"http_url\",\n \"label\": \"HTTP.URL\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_host\",\n \"label\": \"HTTP.Host\",\n \"doc\": {\n \"format\": {\n \"functions\": \"sub_domain\",\n \"appendTo\": \"http_domain\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_domain\",\n \"label\": \"HTTP.Domain\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_line\",\n \"label\": \"HTTP.Request Line\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_line\",\n \"label\": \"HTTP.Response Line\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_header\",\n \"label\": \"HTTP.Request Headers\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_header\",\n \"label\": \"HTTP.Response Headers\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_content\",\n \"label\": \"HTTP.Request Content\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_content\",\n \"label\": \"HTTP.Response Content\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_body\",\n \"label\": \"HTTP.Request Body\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"file\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_body\",\n \"label\": \"HTTP.Response Body\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"file\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_body_key\",\n \"label\": \"HTTP.Request Body Key\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_body_key\",\n \"label\": \"HTTP.Response Body Key\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_proxy_flag\",\n \"label\": \"HTTP.Proxy Flag\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"http_sequence\",\n \"label\": \"HTTP.Sequence\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"http_snapshot\",\n \"label\": \"HTTP.Snapshot\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_cookie\",\n \"label\": \"HTTP.Cookie\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_referer\",\n \"label\": \"HTTP.Referer\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_user_agent\",\n \"label\": \"HTTP.User Agent\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_content_length\",\n \"label\": \"HTTP.Content Length\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_content_type\",\n \"label\": \"HTTP.Content Type\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_set_cookie\",\n \"label\": \"HTTP.Set Cookie\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_version\",\n \"label\": \"HTTP.Version\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_lantency_ms\",\n \"label\": \"HTTP.Response Latency(ms)\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"http_session_duration_ms\",\n \"label\": \"HTTP.Session Duration(ms)\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"http_action_file_size\",\n \"label\": \"HTTP.Action File Size\",\n \"type\": \"int\"\n },\n {\n \"name\": \"mail_protocol_type\",\n \"label\": \"Mail.Protocol Type\",\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_account\",\n \"label\": \"Mail.Account\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_from_cmd\",\n \"label\": \"Mail.From CMD\",\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_to_cmd\",\n \"label\": \"Mail.To CMD\",\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_from\",\n \"label\": \"Mail.From\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"email\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_to\",\n \"label\": \"Mail.To\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"email\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_cc\",\n \"label\": \"Mail.CC\",\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_bcc\",\n \"label\": \"Mail.BCC\",\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_subject\",\n \"label\": \"Mail.Subject\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"decode_of_base64\",\n \"param\": \"$.mail_subject_charset\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_subject_charset\",\n \"label\": \"Mail.Subject Charset\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_content\",\n \"label\": \"Mail.Content\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_content_charset\",\n \"label\": \"Mail.Content Charset\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_attachment_name\",\n \"label\": \"Mail.Attachment\",\n \"doc\": {\n \"format\": {\n \"functions\": \"decode_of_base64\",\n \"param\": \"$.mail_attachment_name_charset\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_attachment_name_charset\",\n \"label\": \"Mail.Attachment Charset\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_attachment_content\",\n \"label\": \"Mail.Attachment Content\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_eml_file\",\n \"label\": \"Mail.EML File\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"file\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_snapshot\",\n \"label\": \"Mail.Snapshot\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"dns_message_id\",\n \"label\": \"DNS.Message ID\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_qr\",\n \"label\": \"DNS.QR\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"QUERY\"\n },\n {\n \"code\": \"1\",\n \"value\": \"REESPONSE\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_opcode\",\n \"label\": \"DNS.OPCODE\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"QUERY\"\n },\n {\n \"code\": \"1\",\n \"value\": \"IQUERY\"\n },\n {\n \"code\": \"2\",\n \"value\": \"STATUS\"\n },\n {\n \"code\": \"5\",\n \"value\": \"UPDATE\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_aa\",\n \"label\": \"DNS.AA\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_tc\",\n \"label\": \"DNS.TC\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_rd\",\n \"label\": \"DNS.RD\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_ra\",\n \"label\": \"DNS.RA\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_rcode\",\n \"label\": \"DNS.RCODE\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_qdcount\",\n \"label\": \"DNS.QDCOUNT\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_ancount\",\n \"label\": \"DNS.ANCOUNT\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_nscount\",\n \"label\": \"DNS.NSCOUNT\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_arcount\",\n \"label\": \"DNS.ARCOUNT\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_qname\",\n \"label\": \"DNS.QNAME\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"dns_qtype\",\n \"label\": \"DNS.QTYPE\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"A\"\n },\n {\n \"code\": \"2\",\n \"value\": \"NS\"\n },\n {\n \"code\": \"5\",\n \"value\": \"CNAME\"\n },\n {\n \"code\": \"6\",\n \"value\": \"SOA\"\n },\n {\n \"code\": \"11\",\n \"value\": \"WKS\"\n },\n {\n \"code\": \"12\",\n \"value\": \"PTR\"\n },\n {\n \"code\": \"13\",\n \"value\": \"HINFO\"\n },\n {\n \"code\": \"11\",\n \"value\": \"WKS\"\n },\n {\n \"code\": \"15\",\n \"value\": \"MX\"\n },\n {\n \"code\": \"28\",\n \"value\": \"AAAA\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_qclass\",\n \"label\": \"DNS.QCLASS\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_cname\",\n \"label\": \"DNS.CNAME\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"dns_sub\",\n \"label\": \"DNS.SUB\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"DNS\"\n },\n {\n \"code\": \"2\",\n \"value\": \"DNSSEC\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_rr\",\n \"label\": \"DNS.RR\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_version\",\n \"label\": \"SSL.Version\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_sni\",\n \"label\": \"SSL.SNI\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"sub_domain\",\n \"appendTo\": \"http_domain\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_san\",\n \"label\": \"SSL.SAN\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_cn\",\n \"label\": \"SSL.CN\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_pinningst\",\n \"label\": \"SSL.Pinning\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"Not Pinning\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Pinning\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Maybe Pinning\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_intercept_state\",\n \"label\": \"SSL.Intercept State\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"Passthrough\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Intercept\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Shutdown\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_server_side_latency\",\n \"label\": \"SSL.Server Side Latency(ms)\",\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_client_side_latency\",\n \"label\": \"SSL.Client Side Latency(ms)\",\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_server_side_version\",\n \"label\": \"SSL.Server Side Version\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_client_side_version\",\n \"label\": \"SSL.Client Side Version\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_cert_verify\",\n \"label\": \"SSL.Certificate Verify\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"No\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Yes\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_error\",\n \"label\": \"SSL.Error\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_con_latency_ms\",\n \"label\": \"SSL.Connection Latency(ms)\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_ja3_fingerprint\",\n \"label\": \"SSL.JA3\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_ja3_hash\",\n \"label\": \"SSL.JA3 hash\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_cert_issuer\",\n \"label\": \"SSL.Issuer\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_cert_subject\",\n \"label\": \"SSL.Subject\",\n \"type\": \"string\"\n },\n {\n \"name\": \"quic_version\",\n \"label\": \"QUIC.Version\",\n \"type\": \"string\"\n },\n {\n \"name\": \"quic_sni\",\n \"label\": \"QUIC.SNI\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"sub_domain\",\n \"appendTo\": \"http_domain\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"quic_user_agent\",\n \"label\": \"QUIC.User Agent\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ftp_account\",\n \"label\": \"FTP.Account\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ftp_url\",\n \"label\": \"FTP.URL\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ftp_content\",\n \"label\": \"FTP.Content\",\n \"type\": \"string\"\n },\n {\n \"name\": \"bgp_type\",\n \"label\": \"BGP.Type\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"bgp_as_num\",\n \"label\": \"BGP.AS Number\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"bgp_route\",\n \"label\": \"BGP.Route\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"voip_calling_account\",\n \"label\": \"VoIP.Calling Account\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"voip_called_account\",\n \"label\": \"VoIP.Called Account\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"voip_calling_number\",\n \"label\": \"VoIP.Calling Number\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"voip_called_number\",\n \"label\": \"VoIP.Called Number\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"streaming_media_url\",\n \"label\": \"Streaming.Media URL\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"streaming_media_protocol\",\n \"label\": \"Streaming.Media Protocol\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"app_extra_info\",\n \"label\": \"APP.Extra Info\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_call_id\",\n \"label\": \"SIP.Call-ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_originator_description\",\n \"label\": \"SIP.Originator\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_responder_description\",\n \"label\": \"SIP.Responder\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_user_agent\",\n \"label\": \"SIP.User-Agent\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_server\",\n \"label\": \"SIP.Server\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_originator_sdp_connect_ip\",\n \"label\": \"SIP.Originator IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_originator_sdp_media_port\",\n \"label\": \"SIP.Originator Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"sip_originator_sdp_media_type\",\n \"label\": \"SIP.Originator Media Type\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_originator_sdp_content\",\n \"label\": \"SIP.Originator Content\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_responder_sdp_connect_ip\",\n \"label\": \"SIP.Responder IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_responder_sdp_media_port\",\n \"label\": \"SIP.Responder Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"sip_responder_sdp_media_type\",\n \"label\": \"SIP.Responder Media Type\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_responder_sdp_content\",\n \"label\": \"SIP.Responder Content\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_duration\",\n \"label\": \"SIP.Duration\",\n \"type\": \"int\"\n },\n {\n \"name\": \"sip_bye\",\n \"label\": \"SIP.Bye\",\n \"type\": \"string\"\n },\n {\n \"name\": \"rtp_payload_type_c2s\",\n \"label\": \"RTP.Payload Type(c2s)\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"PCMU\"\n },\n {\n \"code\": \"1\",\n \"value\": \"1016\"\n },\n {\n \"code\": \"2\",\n \"value\": \"G721\"\n },\n {\n \"code\": \"3\",\n \"value\": \"GSM\"\n },\n {\n \"code\": \"4\",\n \"value\": \"G723\"\n },\n {\n \"code\": \"5\",\n \"value\": \"DVI4_8000\"\n },\n {\n \"code\": \"6\",\n \"value\": \"DVI4_16000\"\n },\n {\n \"code\": \"7\",\n \"value\": \"LPC\"\n },\n {\n \"code\": \"8\",\n \"value\": \"PCMA\"\n },\n {\n \"code\": \"9\",\n \"value\": \"G722\"\n },\n {\n \"code\": \"10\",\n \"value\": \"L16_STEREO\"\n },\n {\n \"code\": \"11\",\n \"value\": \"L16_MONO\"\n },\n {\n \"code\": \"12\",\n \"value\": \"QCELP\"\n },\n {\n \"code\": \"13\",\n \"value\": \"CN\"\n },\n {\n \"code\": \"14\",\n \"value\": \"MPA\"\n },\n {\n \"code\": \"15\",\n \"value\": \"G728\"\n },\n {\n \"code\": \"16\",\n \"value\": \"DVI4_11025\"\n },\n {\n \"code\": \"17\",\n \"value\": \"DVI4_22050\"\n },\n {\n \"code\": \"18\",\n \"value\": \"G729\"\n },\n {\n \"code\": \"19\",\n \"value\": \"CN_OLD\"\n },\n {\n \"code\": \"25\",\n \"value\": \"CELB\"\n },\n {\n \"code\": \"26\",\n \"value\": \"JPEG\"\n },\n {\n \"code\": \"28\",\n \"value\": \"NV\"\n },\n {\n \"code\": \"31\",\n \"value\": \"H261\"\n },\n {\n \"code\": \"32\",\n \"value\": \"MPV\"\n },\n {\n \"code\": \"33\",\n \"value\": \"MP2T\"\n },\n {\n \"code\": \"34\",\n \"value\": \"H263\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"rtp_payload_type_s2c\",\n \"label\": \"RTP.Payload Type(s2c)\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"PCMU\"\n },\n {\n \"code\": \"1\",\n \"value\": \"1016\"\n },\n {\n \"code\": \"2\",\n \"value\": \"G721\"\n },\n {\n \"code\": \"3\",\n \"value\": \"GSM\"\n },\n {\n \"code\": \"4\",\n \"value\": \"G723\"\n },\n {\n \"code\": \"5\",\n \"value\": \"DVI4_8000\"\n },\n {\n \"code\": \"6\",\n \"value\": \"DVI4_16000\"\n },\n {\n \"code\": \"7\",\n \"value\": \"LPC\"\n },\n {\n \"code\": \"8\",\n \"value\": \"PCMA\"\n },\n {\n \"code\": \"9\",\n \"value\": \"G722\"\n },\n {\n \"code\": \"10\",\n \"value\": \"L16_STEREO\"\n },\n {\n \"code\": \"11\",\n \"value\": \"L16_MONO\"\n },\n {\n \"code\": \"12\",\n \"value\": \"QCELP\"\n },\n {\n \"code\": \"13\",\n \"value\": \"CN\"\n },\n {\n \"code\": \"14\",\n \"value\": \"MPA\"\n },\n {\n \"code\": \"15\",\n \"value\": \"G728\"\n },\n {\n \"code\": \"16\",\n \"value\": \"DVI4_11025\"\n },\n {\n \"code\": \"17\",\n \"value\": \"DVI4_22050\"\n },\n {\n \"code\": \"18\",\n \"value\": \"G729\"\n },\n {\n \"code\": \"19\",\n \"value\": \"CN_OLD\"\n },\n {\n \"code\": \"25\",\n \"value\": \"CELB\"\n },\n {\n \"code\": \"26\",\n \"value\": \"JPEG\"\n },\n {\n \"code\": \"28\",\n \"value\": \"NV\"\n },\n {\n \"code\": \"31\",\n \"value\": \"H261\"\n },\n {\n \"code\": \"32\",\n \"value\": \"MPV\"\n },\n {\n \"code\": \"33\",\n \"value\": \"MP2T\"\n },\n {\n \"code\": \"34\",\n \"value\": \"H263\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"rtp_pcap_path\",\n \"label\": \"RTP.PCAP\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"files\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"rtp_originator_dir\",\n \"label\": \"RTP.Direction\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"unknown\"\n },\n {\n \"code\": \"1\",\n \"value\": \"c2s\"\n },\n {\n \"code\": \"2\",\n \"value\": \"s2c\"\n }\n ]\n },\n \"type\": \"int\"\n }\n ]\n}', '941525cd62efbac81761a9c86f4cf630', '2021-06-01 21:56:44', '2021-06-01 21:56:44', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (21, 'meta_data.json', 'Galaxy', '[\n {\n \"namespace\":\"tsg_galaxy_v3\",\n \"group\":\"CLICKHOUSE_GROUP\",\n \"tables\":[\n \"connection_record_log\",\n \"radius_record_log\",\n \"connection_record_log_common_subscriber_id\",\n \"proxy_event_log\",\n \"security_event_log\",\n \"connection_record_log_http_domain\",\n \"sys_packet_capture_log\",\n \"radius_onff_log\",\n \"connection_record_log_common_server_ip\",\n \"connection_record_log_common_client_ip\",\n \"voip_record_log\",\n \"transaction_record_log\",\n \"gtpc_record_log\",\n \"live_session_record_log\"\n ]\n },\n {\n \"namespace\":\"system\",\n \"group\":\"CLICKHOUSE_GROUP\",\n \"tables\":[\n \"query_log_cluster\",\n \"tables_cluster\",\n \"columns_cluster\",\n \"disks_cluster\",\n \"parts_cluster\",\n \"processes\",\n \"query_log\"\n ]\n },\n {\n \"namespace\":\"druid\",\n \"group\":\"DRUID_GROUP\",\n \"tables\":[\n \"top_internal_host_log\",\n \"top_website_domain_log\",\n \"proxy_event_hits_log\",\n \"sys_storage_log\",\n \"security_event_hits_log\",\n \"traffic_protocol_stat_log\",\n \"top_server_ip_log\",\n \"traffic_summary_log\",\n \"traffic_metrics_log\",\n \"top_user_log\",\n \"top_urls_log\",\n \"top_client_ip_log\",\n \"top_external_host_log\"\n ]\n },\n {\n \"namespace\":\"etl\",\n \"group\":\"ETL_GROUP\",\n \"tables\":[\n \"liveChart_history\",\n \"liveChart_live\"\n ]\n }\n]', '9b832a586389998aa5129e8c9a39c3a8', '2021-06-01 21:56:45', '2021-06-02 00:53:59', 'nacos', '192.168.32.48', 'galaxy-qgw-service', 'prod', '', '', '', 'json', ''); +INSERT INTO `config_info` VALUES (22, 'parts_cluster.json', 'Galaxy', '{\n \"namespace\": \"system\",\n \"type\": \"record\",\n \"name\": \"parts_cluster\",\n \"fields\": [\n {\n \"name\": \"name\",\n \"type\": \"string\"\n }\n ]\n}', 'b0bb0813630aef2994923e57e17fd16c', '2021-06-01 21:56:46', '2021-06-01 21:56:46', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (23, 'processes.json', 'Galaxy', '{\n \"namespace\": \"system\",\n \"type\": \"record\",\n \"name\": \"processes\",\n \"fields\": [\n {\n \"name\": \"query_id\",\n \"type\": \"string\"\n }\n ]\n}', '68a220d85b90f21dcb94e6e220e9b74e', '2021-06-01 21:56:47', '2021-06-01 21:56:47', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (24, 'proxy_event_hits_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"proxy_event_hits_log\",\n \"namespace\": \"druid\",\n \"doc\": {\n \"partition_key\": \"__time\"\n },\n \"fields\": [\n {\n \"name\": \"__time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"action\",\n \"type\": \"long\"\n },\n {\n \"name\": \"isp\",\n \"type\": \"string\"\n },\n {\n \"name\": \"entrance_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"hits\",\n \"type\": \"long\"\n },\n {\n \"name\": \"policy_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"sub_action\",\n \"type\": \"string\"\n },\n {\n \"name\": \"country\",\n \"type\": \"string\"\n },\n {\n \"name\": \"location\",\n \"type\": \"string\"\n },\n {\n \"name\": \"c2s_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"s2c_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"ip_object\",\n \"type\": \"string\"\n }\n ]\n}', 'c9b100dc48794be516ead521ce6c2200', '2021-06-01 21:56:47', '2021-06-01 21:56:47', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (25, 'proxy_event_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"proxy_event_log\",\n \"namespace\": \"tsg_galaxy_v3\",\n \"doc\": {\n \"primary_key\": \"common_log_id\",\n \"partition_key\": \"common_recv_time\",\n \"functions\": {\n \"aggregation\": [\n {\n \"name\": \"COUNT\",\n \"label\": \"COUNT\",\n \"function\": \"count(expr)\"\n },\n {\n \"name\": \"COUNT_DISTINCT\",\n \"label\": \"COUNT_DISTINCT\",\n \"function\": \"count(distinct expr)\"\n },\n {\n \"name\": \"AVG\",\n \"label\": \"AVG\",\n \"function\": \"avg(expr)\"\n },\n {\n \"name\": \"SUM\",\n \"label\": \"SUM\",\n \"function\": \"sum(expr)\"\n },\n {\n \"name\": \"MAX\",\n \"label\": \"MAX\",\n \"function\": \"max(expr)\"\n },\n {\n \"name\": \"MIN\",\n \"label\": \"MIN\",\n \"function\": \"min(expr)\"\n }\n ],\n \"operator\": [\n {\n \"name\": \"=\",\n \"label\": \"=\",\n \"function\": \"expr = value\"\n },\n {\n \"name\": \"!=\",\n \"label\": \"!=\",\n \"function\": \"expr != value\"\n },\n {\n \"name\": \">\",\n \"label\": \">\",\n \"function\": \"expr > value\"\n },\n {\n \"name\": \"<\",\n \"label\": \"<\",\n \"function\": \"expr < value\"\n },\n {\n \"name\": \">=\",\n \"label\": \">=\",\n \"function\": \"expr >= value\"\n },\n {\n \"name\": \"<=\",\n \"label\": \"<=\",\n \"function\": \"expr <= value\"\n },\n {\n \"name\": \"in\",\n \"label\": \"IN\",\n \"function\": \"expr in (values)\"\n },\n {\n \"name\": \"not in\",\n \"label\": \"NOT IN\",\n \"function\": \"expr not in (values)\"\n },\n {\n \"name\": \"like\",\n \"label\": \"LIKE\",\n \"function\": \"expr like value\"\n },\n {\n \"name\": \"not like\",\n \"label\": \"NOT LIKE\",\n \"function\": \"expr not like value\"\n },\n {\n \"name\": \"notEmpty\",\n \"label\": \"NOT EMPTY\",\n \"function\": \"notEmpty(expr)\"\n },\n {\n \"name\": \"empty\",\n \"label\": \"EMPTY\",\n \"function\": \"empty(expr)\"\n }\n ]\n },\n \"schema_query\": {\n \"dimensions\": [\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_policy_id\",\n \"common_sub_action\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_client_location\",\n \"common_server_location\",\n \"common_subscriber_id\",\n \"common_client_port\",\n \"common_server_port\",\n \"common_schema_type\",\n \"common_data_center\",\n \"common_client_asn\",\n \"common_server_asn\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"http_host\",\n \"http_domain\",\n \"http_url\",\n \"doh_host\",\n \"doh_qname\"\n ],\n \"metrics\": [\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_subscriber_id\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_sessions\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"http_host\",\n \"http_domain\",\n \"http_url\",\n \"doh_host\",\n \"doh_qname\"\n ],\n \"filters\": [\n \"common_policy_id\",\n \"common_sub_action\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_client_port\",\n \"common_server_port\",\n \"common_client_location\",\n \"common_server_location\",\n \"common_subscriber_id\",\n \"common_l4_protocol\",\n \"common_data_center\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_client_asn\",\n \"common_server_asn\",\n \"common_direction\",\n \"common_schema_type\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"http_host\",\n \"http_domain\",\n \"http_url\",\n \"http_content_type\",\n \"doh_host\",\n \"doh_qname\"\n ],\n \"references\": {\n \"aggregation\": [\n {\n \"type\": \"int\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"long\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"float\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"double\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"string\",\n \"functions\": \"COUNT,COUNT_DISTINCT\"\n },\n {\n \"type\": \"date\",\n \"functions\": \"COUNT,COUNT_DISTINCT,MAX,MIN\"\n },\n {\n \"type\": \"timestamp\",\n \"functions\": \"COUNT,COUNT_DISTINCT,MAX,MIN\"\n }\n ],\n \"operator\": [\n {\n \"type\": \"int\",\n \"functions\": \"=,!=,>,<,>=,<=,in,not in\"\n },\n {\n \"type\": \"long\",\n \"functions\": \"=,!=,>,<,>=,<=,in,not in\"\n },\n {\n \"type\": \"float\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"double\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"string\",\n \"functions\": \"=,!=,in,not in,like,not like,notEmpty,empty\"\n },\n {\n \"type\": \"date\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"timestamp\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n }\n ]\n },\n \"details\": {\n \"general\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_stream_trace_id\",\n \"common_direction\",\n \"common_stream_dir\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_processing_time\",\n \"common_entrance_id\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_data_center\",\n \"common_sled_ip\"\n ],\n \"action\": [\n \"common_action\",\n \"common_sub_action\",\n \"common_policy_id\",\n \"common_user_tags\",\n \"common_user_region\"\n ],\n \"source\": [\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_client_port\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\"\n ],\n \"destination\": [\n \"common_server_ip\",\n \"common_external_ip\",\n \"common_server_port\",\n \"common_server_location\",\n \"common_server_asn\"\n ],\n \"application\": [\n \"common_app_id\",\n \"common_app_label\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_protocol_label\",\n \"common_service_category\",\n \"common_service\",\n \"common_l4_protocol\"\n ],\n \"transmission\": [\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_first_ttl\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\"\n ],\n \"other\": [\n \"common_address_type\",\n \"common_schema_type\",\n \"common_device_tag\",\n \"common_encapsulation\",\n \"common_tunnels\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\"\n ]\n }\n },\n \"schema_type\": {\n \"HTTP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"http_url\",\n \"http_host\",\n \"http_domain\",\n \"http_request_line\",\n \"http_response_line\",\n \"http_request_header\",\n \"http_response_header\",\n \"http_request_content\",\n \"http_response_content\",\n \"http_request_body\",\n \"http_response_body\",\n \"http_request_body_key\",\n \"http_response_body_key\",\n \"http_proxy_flag\",\n \"http_sequence\",\n \"http_snapshot\",\n \"http_cookie\",\n \"http_referer\",\n \"http_user_agent\",\n \"http_content_length\",\n \"http_content_type\",\n \"http_set_cookie\",\n \"http_version\",\n \"http_response_lantency_ms\",\n \"http_session_duration_ms\",\n \"http_action_file_size\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"http_url\",\n \"common_sub_action\"\n ]\n },\n \"DoH\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"doh_url\",\n \"doh_host\",\n \"doh_request_line\",\n \"doh_response_line\",\n \"doh_cookie\",\n \"doh_referer\",\n \"doh_user_agent\",\n \"doh_content_length\",\n \"doh_content_type\",\n \"doh_set_cookie\",\n \"doh_version\",\n \"doh_message_id\",\n \"doh_qr\",\n \"doh_opcode\",\n \"doh_aa\",\n \"doh_tc\",\n \"doh_rd\",\n \"doh_ra\",\n \"doh_rcode\",\n \"doh_qdcount\",\n \"doh_ancount\",\n \"doh_nscount\",\n \"doh_arcount\",\n \"doh_qname\",\n \"doh_qtype\",\n \"doh_qclass\",\n \"doh_cname\",\n \"doh_sub\",\n \"doh_rr\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_client_ip\",\n \"doh_url\",\n \"doh_qname\",\n \"common_server_port\"\n ]\n }\n },\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_sub_action\",\n \"common_schema_type\"\n ],\n \"tunnel_type\": {\n \"GTP\": [\n {\n \"name\": \"gtp_sgw_ip\",\n \"label\": \"S-GW IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_pgw_ip\",\n \"label\": \"P-GW IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_sgw_port\",\n \"label\": \"S-GW Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"gtp_pgw_port\",\n \"label\": \"P-GW Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"gtp_uplink_teid\",\n \"label\": \"Uplink TEID\",\n \"type\": \"long\"\n },\n {\n \"name\": \"gtp_downlink_teid\",\n \"label\": \"Downlink TEID\",\n \"type\": \"long\"\n }\n ],\n \"MPLS\": [\n {\n \"name\": \"mpls_c2s_direction_label\",\n \"label\": \"Multiprotocol Label(c2s)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n },\n {\n \"name\": \"mpls_s2c_direction_label\",\n \"label\": \"Multiprotocol Label(s2c)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n }\n ],\n \"VLAN\": [\n {\n \"name\": \"vlan_c2s_direction_id\",\n \"label\": \"VLAN Direction(c2s)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n },\n {\n \"name\": \"vlan_s2c_direction_id\",\n \"label\": \"VLAN Direction(s2c)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n }\n ],\n \"ETHERNET\": [\n {\n \"name\": \"source_mac\",\n \"label\": \"Source MAC\",\n \"type\": \"string\"\n },\n {\n \"name\": \"destination_mac\",\n \"label\": \"Destination MAC\",\n \"type\": \"string\"\n }\n ],\n \"MULTIPATH_ETHERNET\": [\n {\n \"name\": \"c2s_source_mac\",\n \"label\": \"Source MAC(c2s)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"c2s_destination_mac\",\n \"label\": \"Destination MAC(c2s)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_source_mac\",\n \"label\": \"Source MAC(s2c)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_destination_mac\",\n \"label\": \"Destination MAC(s2c)\",\n \"type\": \"string\"\n }\n ]\n }\n },\n \"fields\": [\n {\n \"name\": \"common_recv_time\",\n \"label\": \"Receive Time\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_log_id\",\n \"label\": \"Log ID\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"snowflake_id\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_policy_id\",\n \"label\": \"Policy ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_subscriber_id\",\n \"label\": \"Subscriber ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_imei\",\n \"label\": \"IMEI\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_imsi\",\n \"label\": \"IMSI\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_phone_number\",\n \"label\": \"Phone Number\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_ip\",\n \"label\": \"Client IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"geo_asn,radius_match\",\n \"appendTo\": \"common_client_asn,common_subscriber_id\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_internal_ip\",\n \"label\": \"Internal IP\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"if\",\n \"param\": \"$.common_direction=69,$.common_client_ip,$.common_server_ip\"\n },\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_port\",\n \"label\": \"Client Port\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_l4_protocol\",\n \"label\": \"L4 Protocol\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_address_type\",\n \"label\": \"Address Type\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"4\",\n \"value\": \"ipv4\"\n },\n {\n \"code\": \"6\",\n \"value\": \"ipv6\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_server_ip\",\n \"label\": \"Server IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"geo_asn\",\n \"appendTo\": \"common_server_asn\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_port\",\n \"label\": \"Server Port\",\n \"type\": \"int\",\n \"doc\": {\n \"allow_query\": \"true\"\n }\n },\n {\n \"name\": \"common_external_ip\",\n \"label\": \"External IP\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"if\",\n \"param\": \"$.common_direction=73,$.common_client_ip,$.common_server_ip\"\n },\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_action\",\n \"label\": \"Action\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"None\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Intercept\"\n },\n {\n \"code\": \"16\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"48\",\n \"value\": \"Manipulation\"\n },\n {\n \"code\": \"128\",\n \"value\": \"Allow\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_direction\",\n \"label\": \"Direction\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"69\",\n \"value\": \"outbound\"\n },\n {\n \"code\": \"73\",\n \"value\": \"inbound\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_entrance_id\",\n \"label\": \"Entrance ID\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_sled_ip\",\n \"label\": \"Sled IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_location\",\n \"label\": \"Client Location\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_asn\",\n \"label\": \"Client ASN\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_location\",\n \"label\": \"Server Location\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_asn\",\n \"label\": \"Server ASN\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sessions\",\n \"label\": \"Sessions\",\n \"doc\": {\n \"format\": {\n \"functions\": \"set_value\",\n \"param\": \"1\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_num\",\n \"label\": \"Packets Sent\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_num\",\n \"label\": \"Packets Received\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_num\",\n \"label\": \"Bytes Sent\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_num\",\n \"label\": \"Bytes Received\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_diff\",\n \"label\": \"Packets Sent(Diff)\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_diff\",\n \"label\": \"Packets Received(Diff)\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_diff\",\n \"label\": \"Bytes Sent(Diff)\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_diff\",\n \"label\": \"Bytes Received(Diff)\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_service\",\n \"label\": \"Service\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_schema_type\",\n \"label\": \"Schema Type\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"HTTP\",\n \"value\": \"HTTP\"\n },\n {\n \"code\": \"DoH\",\n \"value\": \"DoH\"\n }\n ]\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_user_tags\",\n \"label\": \"User Tags\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sub_action\",\n \"label\": \"Sub Action\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"allow\",\n \"value\": \"Allow\"\n },\n {\n \"code\": \"deny\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"monitor\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"replace\",\n \"value\": \"Replace\"\n },\n {\n \"code\": \"redirect\",\n \"value\": \"Redirect\"\n },\n {\n \"code\": \"insert\",\n \"value\": \"Insert\"\n },\n {\n \"code\": \"hijack\",\n \"value\": \"Hijack\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_user_region\",\n \"label\": \"User Region\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_device_id\",\n \"label\": \"Device ID\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_link_id\",\n \"label\": \"Link ID\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_isp\",\n \"label\": \"ISP\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_device_tag\",\n \"label\": \"Device Tag\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_data_center\",\n \"label\": \"Data Center\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"City A\",\n \"value\": \"City A\"\n },\n {\n \"code\": \"City B\",\n \"value\": \"City B\"\n },\n {\n \"code\": \"City C\",\n \"value\": \"City C\"\n },\n {\n \"code\": \"City D\",\n \"value\": \"City D\"\n },\n {\n \"code\": \"City E\",\n \"value\": \"City E\"\n },\n {\n \"code\": \"City F\",\n \"value\": \"City F\"\n },\n {\n \"code\": \"City G\",\n \"value\": \"City G\"\n },\n {\n \"code\": \"City H\",\n \"value\": \"City H\"\n },\n {\n \"code\": \"City I\",\n \"value\": \"City I\"\n },\n {\n \"code\": \"City J\",\n \"value\": \"City J\"\n },\n {\n \"code\": \"City K\",\n \"value\": \"City K\"\n },\n {\n \"code\": \"City L\",\n \"value\": \"City L\"\n },\n {\n \"code\": \"City M\",\n \"value\": \"City M\"\n },\n {\n \"code\": \"City N\",\n \"value\": \"City N\"\n },\n {\n \"code\": \"City O\",\n \"value\": \"City O\"\n },\n {\n \"code\": \"City P\",\n \"value\": \"City P\"\n },\n {\n \"code\": \"City Q\",\n \"value\": \"City Q\"\n },\n {\n \"code\": \"City R\",\n \"value\": \"City R\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_encapsulation\",\n \"label\": \"Encapsulation\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"Ethernet\"\n },\n {\n \"code\": \"8\",\n \"value\": \"PPP\"\n },\n {\n \"code\": \"12\",\n \"value\": \"CiscoHDLC\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_app_label\",\n \"label\": \"Application Label\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_tunnels\",\n \"label\": \"Tunnels\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_protocol_label\",\n \"label\": \"Protocol Label\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_app_id\",\n \"label\": \"Application ID\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_app_surrogate_id\",\n \"label\": \"Surrogate ID\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_l7_protocol\",\n \"label\": \"L7 Protocol\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_service_category\",\n \"label\": \"FQDN Category\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"dict_location\": {\n \"path\": \"/v1/category/dict\",\n \"key\": \"categoryId\",\n \"value\": \"categoryName\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_start_time\",\n \"label\": \"Start Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_end_time\",\n \"label\": \"End Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"get_value\",\n \"appendTo\": \"common_recv_time\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_establish_latency_ms\",\n \"label\": \"Establish Latency(ms)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_con_duration_ms\",\n \"label\": \"Duration(ms)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_stream_dir\",\n \"label\": \"Stream Direction\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"c2s\"\n },\n {\n \"code\": \"2\",\n \"value\": \"s2c\"\n },\n {\n \"code\": \"3\",\n \"value\": \"double\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_address_list\",\n \"label\": \"Address List\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_has_dup_traffic\",\n \"label\": \"Duplication Traffic\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"No\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Yes\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_stream_error\",\n \"label\": \"Stream Error\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_stream_trace_id\",\n \"label\": \"Session ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_link_info_c2s\",\n \"label\": \"Link Info(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_link_info_s2c\",\n \"label\": \"Link Info(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_c2s_ipfrag_num\",\n \"label\": \"Fragmentation Packets(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_ipfrag_num\",\n \"label\": \"Fragmentation Packets(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_tcp_lostlen\",\n \"label\": \"Sequence Gap Loss(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_tcp_lostlen\",\n \"label\": \"Sequence Gap Loss(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_tcp_unorder_num\",\n \"label\": \"Unorder Packets(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_tcp_unorder_num\",\n \"label\": \"Unorder Packets(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_retrans\",\n \"label\": \"Packet Retransmission(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_retrans\",\n \"label\": \"Packet Retransmission(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_retrans\",\n \"label\": \"Byte Retransmission(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_retrans\",\n \"label\": \"Byte Retransmission(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_tcp_client_isn\",\n \"label\": \"TCP Client ISN\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_tcp_server_isn\",\n \"label\": \"TCP Server ISN\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_first_ttl\",\n \"label\": \"First TTL\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_processing_time\",\n \"label\": \"Processing Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"current_timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"http_url\",\n \"label\": \"HTTP.URL\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_host\",\n \"label\": \"HTTP.Host\",\n \"doc\": {\n \"format\": {\n \"functions\": \"sub_domain\",\n \"appendTo\": \"http_domain\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_domain\",\n \"label\": \"HTTP.Domain\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_line\",\n \"label\": \"HTTP.Request Line\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_line\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"HTTP.Response Line\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_header\",\n \"label\": \"HTTP.Request Header\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_header\",\n \"label\": \"HTTP.Response Header\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_content\",\n \"label\": \"HTTP.Request Content\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_content\",\n \"label\": \"HTTP.Response Content\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_body\",\n \"label\": \"HTTP.Request Body\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"file\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_body\",\n \"label\": \"HTTP.Response Body\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"file\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_body_key\",\n \"label\": \"HTTP.Request Body Key\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_body_key\",\n \"label\": \"HTTP.Response Body Key\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_proxy_flag\",\n \"label\": \"HTTP.Proxy Flag\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"http_sequence\",\n \"label\": \"HTTP.Sequence\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"http_snapshot\",\n \"label\": \"HTTP.Snapshot\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_cookie\",\n \"label\": \"HTTP.Cookie\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_referer\",\n \"label\": \"HTTP.Referer\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_user_agent\",\n \"label\": \"HTTP.User Agent\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_content_length\",\n \"label\": \"HTTP.Content Length\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_content_type\",\n \"label\": \"HTTP.Content Type\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_set_cookie\",\n \"label\": \"HTTP.Set Cookie\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_version\",\n \"label\": \"HTTP.Version\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_lantency_ms\",\n \"label\": \"HTTP.Response Latency(ms)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"http_session_duration_ms\",\n \"label\": \"HTTP.Session Duration(ms)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"http_action_file_size\",\n \"label\": \"HTTP.Action File Size\",\n \"type\": \"int\"\n },\n {\n \"name\": \"doh_url\",\n \"label\": \"DoH.URL\",\n \"type\": \"string\"\n },\n {\n \"name\": \"doh_host\",\n \"label\": \"DoH.Host\",\n \"type\": \"string\"\n },\n {\n \"name\": \"doh_request_line\",\n \"label\": \"DoH.Request Line\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"doh_response_line\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"DoH.Response Line\",\n \"type\": \"string\"\n },\n {\n \"name\": \"doh_cookie\",\n \"label\": \"DoH.Cookie\",\n \"type\": \"string\"\n },\n {\n \"name\": \"doh_referer\",\n \"label\": \"DoH.Referer\",\n \"type\": \"string\"\n },\n {\n \"name\": \"doh_user_agent\",\n \"label\": \"DoH.User Agent\",\n \"type\": \"string\"\n },\n {\n \"name\": \"doh_content_length\",\n \"label\": \"DoH.Content Length\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"doh_content_type\",\n \"label\": \"DoH.Content Type\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"doh_set_cookie\",\n \"label\": \"DoH.Set Cookie\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"doh_version\",\n \"label\": \"DoH.Version\",\n \"type\": \"string\"\n },\n {\n \"name\": \"doh_message_id\",\n \"label\": \"DoH.Message ID\",\n \"type\": \"int\"\n },\n {\n \"name\": \"doh_qr\",\n \"label\": \"DoH.QR\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"QUERY\"\n },\n {\n \"code\": \"1\",\n \"value\": \"REESPONSE\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"doh_opcode\",\n \"label\": \"DoH.OPCODE\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"QUERY\"\n },\n {\n \"code\": \"1\",\n \"value\": \"IQUERY\"\n },\n {\n \"code\": \"2\",\n \"value\": \"STATUS\"\n },\n {\n \"code\": \"5\",\n \"value\": \"UPDATE\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"doh_aa\",\n \"label\": \"DoH.AA\",\n \"type\": \"int\"\n },\n {\n \"name\": \"doh_tc\",\n \"label\": \"DoH.TC\",\n \"type\": \"int\"\n },\n {\n \"name\": \"doh_rd\",\n \"label\": \"DoH.RD\",\n \"type\": \"int\"\n },\n {\n \"name\": \"doh_ra\",\n \"label\": \"DoH.RA\",\n \"type\": \"int\"\n },\n {\n \"name\": \"doh_rcode\",\n \"label\": \"DoH.RCODE\",\n \"type\": \"int\"\n },\n {\n \"name\": \"doh_qdcount\",\n \"label\": \"DoH.QDCOUNT\",\n \"type\": \"int\"\n },\n {\n \"name\": \"doh_ancount\",\n \"label\": \"DoH.ANCOUNT\",\n \"type\": \"int\"\n },\n {\n \"name\": \"doh_nscount\",\n \"label\": \"DoH.NSCOUNT\",\n \"type\": \"int\"\n },\n {\n \"name\": \"doh_arcount\",\n \"label\": \"DoH.ARCOUNT\",\n \"type\": \"int\"\n },\n {\n \"name\": \"doh_qname\",\n \"label\": \"DoH.QNAME\",\n \"type\": \"string\"\n },\n {\n \"name\": \"doh_qtype\",\n \"label\": \"DoH.QTYPE\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"A\"\n },\n {\n \"code\": \"2\",\n \"value\": \"NS\"\n },\n {\n \"code\": \"5\",\n \"value\": \"CNAME\"\n },\n {\n \"code\": \"6\",\n \"value\": \"SOA\"\n },\n {\n \"code\": \"11\",\n \"value\": \"WKS\"\n },\n {\n \"code\": \"12\",\n \"value\": \"PTR\"\n },\n {\n \"code\": \"13\",\n \"value\": \"HINFO\"\n },\n {\n \"code\": \"11\",\n \"value\": \"WKS\"\n },\n {\n \"code\": \"15\",\n \"value\": \"MX\"\n },\n {\n \"code\": \"28\",\n \"value\": \"AAAA\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"doh_qclass\",\n \"label\": \"DoH.QCLASS\",\n \"type\": \"int\"\n },\n {\n \"name\": \"doh_cname\",\n \"label\": \"DoH.CNAME\",\n \"type\": \"string\"\n },\n {\n \"name\": \"doh_sub\",\n \"label\": \"DoH.SUB\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"DNS\"\n },\n {\n \"code\": \"2\",\n \"value\": \"DNSSEC\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"doh_rr\",\n \"label\": \"DoH.RR\",\n \"type\": \"string\"\n }\n ]\n}', '9ebbc6d0fc6df836f0c7a06215133387', '2021-06-01 21:56:48', '2021-06-01 21:56:48', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (26, 'query_log_cluster.json', 'Galaxy', '{\n \"namespace\": \"system\",\n \"type\": \"record\",\n \"name\": \"query_log_cluster\",\n \"fields\": [\n {\n \"name\": \"type\",\n \"type\": \"string\"\n }\n ]\n}', 'afadf9f14bb2c473416826307cc37481', '2021-06-01 21:56:49', '2021-06-01 21:56:49', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (27, 'query_log.json', 'Galaxy', '{\n \"namespace\": \"system\",\n \"type\": \"record\",\n \"name\": \"query_log\",\n \"fields\": [\n {\n \"name\": \"query_id\",\n \"type\": \"string\"\n }\n ]\n}', '5413fe83fd5020c8e369d301cbe3ddc6', '2021-06-01 21:56:50', '2021-06-01 21:56:50', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (28, 'radius_onff_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"radius_onff_log\",\n \"namespace\": \"tsg_galaxy_v3\",\n \"fields\": [\n {\n \"name\": \"event_timestamp\",\n \"label\": \"Event Time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"account\",\n \"label\": \"Account\",\n \"type\": \"string\"\n },\n {\n \"name\": \"framed_ip\",\n \"label\": \"Framed IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"acct_session_id\",\n \"label\": \"Acct Session ID\",\n \"type\": \"string\"\n },\n {\n \"name\": \"acct_status_type\",\n \"label\": \"Acct Status Type\",\n \"type\": \"int\"\n },\n {\n \"name\": \"acct_session_time\",\n \"label\": \"Acct Session Time\",\n \"type\": \"int\"\n }\n ]\n}', 'a130f1615fbd331e82bad23278b45391', '2021-06-01 21:56:51', '2021-06-01 21:56:51', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (29, 'radius_record_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"radius_record_log\",\n \"namespace\": \"tsg_galaxy_v3\",\n \"doc\": {\n \"primary_key\": \"common_log_id\",\n \"partition_key\": \"common_recv_time\",\n \"functions\": {\n \"aggregation\": [\n {\n \"name\": \"COUNT\",\n \"label\": \"COUNT\",\n \"function\": \"count(expr)\"\n },\n {\n \"name\": \"COUNT_DISTINCT\",\n \"label\": \"COUNT_DISTINCT\",\n \"function\": \"count(distinct expr)\"\n },\n {\n \"name\": \"AVG\",\n \"label\": \"AVG\",\n \"function\": \"avg(expr)\"\n },\n {\n \"name\": \"SUM\",\n \"label\": \"SUM\",\n \"function\": \"sum(expr)\"\n },\n {\n \"name\": \"MAX\",\n \"label\": \"MAX\",\n \"function\": \"max(expr)\"\n },\n {\n \"name\": \"MIN\",\n \"label\": \"MIN\",\n \"function\": \"min(expr)\"\n }\n ],\n \"operator\": [\n {\n \"name\": \"=\",\n \"label\": \"=\",\n \"function\": \"expr = value\"\n },\n {\n \"name\": \"!=\",\n \"label\": \"!=\",\n \"function\": \"expr != value\"\n },\n {\n \"name\": \">\",\n \"label\": \">\",\n \"function\": \"expr > value\"\n },\n {\n \"name\": \"<\",\n \"label\": \"<\",\n \"function\": \"expr < value\"\n },\n {\n \"name\": \">=\",\n \"label\": \">=\",\n \"function\": \"expr >= value\"\n },\n {\n \"name\": \"<=\",\n \"label\": \"<=\",\n \"function\": \"expr <= value\"\n },\n {\n \"name\": \"in\",\n \"label\": \"IN\",\n \"function\": \"expr in (values)\"\n },\n {\n \"name\": \"not in\",\n \"label\": \"NOT IN\",\n \"function\": \"expr not in (values)\"\n },\n {\n \"name\": \"like\",\n \"label\": \"LIKE\",\n \"function\": \"expr like value\"\n },\n {\n \"name\": \"not like\",\n \"label\": \"NOT LIKE\",\n \"function\": \"expr not like value\"\n },\n {\n \"name\": \"notEmpty\",\n \"label\": \"NOT EMPTY\",\n \"function\": \"notEmpty(expr)\"\n },\n {\n \"name\": \"empty\",\n \"label\": \"EMPTY\",\n \"function\": \"empty(expr)\"\n }\n ]\n },\n \"schema_query\": {\n \"dimensions\": [\n \"radius_nas_ip\",\n \"radius_framed_ip\",\n \"common_subscriber_id\"\n ],\n \"metrics\": [\n \"radius_framed_ip\",\n \"radius_event_timestamp\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\"\n ],\n \"filters\": [\n \"radius_framed_ip\",\n \"common_subscriber_id\",\n \"radius_packet_type\",\n \"radius_acct_session_id\",\n \"radius_acct_multi_session_id\",\n \"radius_acct_status_type\"\n ],\n \"references\": {\n \"aggregation\": [\n {\n \"type\": \"int\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"long\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"float\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"double\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"string\",\n \"functions\": \"COUNT,COUNT_DISTINCT\"\n },\n {\n \"type\": \"date\",\n \"functions\": \"COUNT,COUNT_DISTINCT,MAX,MIN\"\n },\n {\n \"type\": \"timestamp\",\n \"functions\": \"COUNT,COUNT_DISTINCT,MAX,MIN\"\n }\n ],\n \"operator\": [\n {\n \"type\": \"int\",\n \"functions\": \"=,!=,>,<,>=,<=,in,not in\"\n },\n {\n \"type\": \"long\",\n \"functions\": \"=,!=,>,<,>=,<=,in,not in\"\n },\n {\n \"type\": \"float\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"double\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"string\",\n \"functions\": \"=,!=,in,not in,like,not like,notEmpty,empty\"\n },\n {\n \"type\": \"date\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"timestamp\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n }\n ]\n },\n \"details\": {\n \"general\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_stream_trace_id\",\n \"common_direction\",\n \"common_stream_dir\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_processing_time\",\n \"common_entrance_id\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_data_center\",\n \"common_sled_ip\"\n ],\n \"action\": [\n \"common_action\",\n \"common_sub_action\",\n \"common_policy_id\",\n \"common_user_tags\",\n \"common_user_region\"\n ],\n \"source\": [\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_client_port\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\"\n ],\n \"destination\": [\n \"common_server_ip\",\n \"common_external_ip\",\n \"common_server_port\",\n \"common_server_location\",\n \"common_server_asn\"\n ],\n \"application\": [\n \"common_app_id\",\n \"common_app_label\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_protocol_label\",\n \"common_service_category\",\n \"common_service\",\n \"common_l4_protocol\"\n ],\n \"transmission\": [\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_first_ttl\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\"\n ],\n \"other\": [\n \"common_address_type\",\n \"common_schema_type\",\n \"common_device_tag\",\n \"common_encapsulation\",\n \"common_tunnels\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\"\n ]\n }\n },\n \"schema_type\": {\n \"RADIUS\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"radius_packet_type\",\n \"radius_nas_ip\",\n \"radius_framed_ip\",\n \"radius_account\",\n \"radius_session_timeout\",\n \"radius_idle_timeout\",\n \"radius_acct_status_type\",\n \"radius_acct_terminate_cause\",\n \"radius_event_timestamp\",\n \"radius_nas_port\",\n \"radius_service_type\",\n \"radius_framed_protocol\",\n \"radius_callback_number\",\n \"radius_callback_id\",\n \"radius_termination_action\",\n \"radius_called_station_id\",\n \"radius_calling_station_id\",\n \"radius_acct_delay_time\",\n \"radius_acct_session_id\",\n \"radius_acct_multi_session_id\",\n \"radius_acct_input_octets\",\n \"radius_acct_output_octets\",\n \"radius_acct_input_packets\",\n \"radius_acct_output_packets\",\n \"radius_acct_session_time\",\n \"radius_acct_link_count\",\n \"radius_acct_interim_interval\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"radius_nas_ip\",\n \"radius_framed_ip\",\n \"radius_acct_status_type\"\n ]\n }\n },\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"radius_nas_ip\",\n \"radius_framed_ip\",\n \"radius_acct_status_type\"\n ],\n \"tunnel_type\": {\n \"GTP\": [\n {\n \"name\": \"gtp_sgw_ip\",\n \"label\": \"S-GW IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_pgw_ip\",\n \"label\": \"P-GW IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_sgw_port\",\n \"label\": \"S-GW Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"gtp_pgw_port\",\n \"label\": \"P-GW Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"gtp_uplink_teid\",\n \"label\": \"Uplink TEID\",\n \"type\": \"long\"\n },\n {\n \"name\": \"gtp_downlink_teid\",\n \"label\": \"Downlink TEID\",\n \"type\": \"long\"\n }\n ],\n \"MPLS\": [\n {\n \"name\": \"mpls_c2s_direction_label\",\n \"label\": \"Multiprotocol Label(c2s)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n },\n {\n \"name\": \"mpls_s2c_direction_label\",\n \"label\": \"Multiprotocol Label(s2c)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n }\n ],\n \"VLAN\": [\n {\n \"name\": \"vlan_c2s_direction_id\",\n \"label\": \"VLAN Direction(c2s)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n },\n {\n \"name\": \"vlan_s2c_direction_id\",\n \"label\": \"VLAN Direction(s2c)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n }\n ],\n \"ETHERNET\": [\n {\n \"name\": \"source_mac\",\n \"label\": \"Source MAC\",\n \"type\": \"string\"\n },\n {\n \"name\": \"destination_mac\",\n \"label\": \"Destination MAC\",\n \"type\": \"string\"\n }\n ],\n \"MULTIPATH_ETHERNET\": [\n {\n \"name\": \"c2s_source_mac\",\n \"label\": \"Source MAC(c2s)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"c2s_destination_mac\",\n \"label\": \"Destination MAC(c2s)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_source_mac\",\n \"label\": \"Source MAC(s2c)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_destination_mac\",\n \"label\": \"Destination MAC(s2c)\",\n \"type\": \"string\"\n }\n ]\n }\n },\n \"fields\": [\n {\n \"name\": \"common_recv_time\",\n \"label\": \"Receive Time\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_log_id\",\n \"label\": \"Log ID\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"snowflake_id\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_policy_id\",\n \"label\": \"Policy ID\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_subscriber_id\",\n \"label\": \"Subscriber ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_imei\",\n \"label\": \"IMEI\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_imsi\",\n \"label\": \"IMSI\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_phone_number\",\n \"label\": \"Phone Number\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_ip\",\n \"label\": \"Client IP\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_port\",\n \"label\": \"Client Port\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_internal_ip\",\n \"label\": \"Internal IP\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_l4_protocol\",\n \"label\": \"L4 Protocol\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_address_type\",\n \"label\": \"Address Type\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"4\",\n \"value\": \"ipv4\"\n },\n {\n \"code\": \"6\",\n \"value\": \"ipv6\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_server_ip\",\n \"label\": \"Server IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"geo_asn\",\n \"appendTo\": \"common_server_asn\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_port\",\n \"label\": \"Server Port\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_external_ip\",\n \"label\": \"External IP\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_action\",\n \"label\": \"Action\",\n \"doc\": {\n \"visibility\": \"hidden\",\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"None\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Intercept\"\n },\n {\n \"code\": \"16\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"48\",\n \"value\": \"Manipulation\"\n },\n {\n \"code\": \"128\",\n \"value\": \"Allow\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_direction\",\n \"label\": \"Direction\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"69\",\n \"value\": \"outbound\"\n },\n {\n \"code\": \"73\",\n \"value\": \"inbound\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_entrance_id\",\n \"label\": \"Entrance ID\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_sled_ip\",\n \"label\": \"Sled IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_location\",\n \"label\": \"Client Location\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_asn\",\n \"label\": \"Client ASN\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_location\",\n \"label\": \"Server Location\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_asn\",\n \"label\": \"Server ASN\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sessions\",\n \"label\": \"Sessions\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_num\",\n \"label\": \"Packets Sent\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_num\",\n \"label\": \"Packets Received\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_num\",\n \"label\": \"Bytes Sent\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_num\",\n \"label\": \"Bytes Received\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_diff\",\n \"label\": \"Packets Sent(Diff)\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_diff\",\n \"label\": \"Packets Received(Diff)\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_diff\",\n \"label\": \"Bytes Sent(Diff)\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_diff\",\n \"label\": \"Bytes Received(Diff)\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_service\",\n \"label\": \"Service\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_schema_type\",\n \"label\": \"Schema Type\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"BASE\",\n \"value\": \"BASE\"\n },\n {\n \"code\": \"HTTP\",\n \"value\": \"HTTP\"\n },\n {\n \"code\": \"MAIL\",\n \"value\": \"MAIL\"\n },\n {\n \"code\": \"DNS\",\n \"value\": \"DNS\"\n },\n {\n \"code\": \"SSL\",\n \"value\": \"SSL\"\n },\n {\n \"code\": \"FTP\",\n \"value\": \"FTP\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_user_tags\",\n \"label\": \"User Tags\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sub_action\",\n \"label\": \"Sub Action\",\n \"doc\": {\n \"data\": [\n {\n \"code\": \"allow\",\n \"value\": \"Allow\"\n },\n {\n \"code\": \"deny\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"monitor\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"replace\",\n \"value\": \"Replace\"\n },\n {\n \"code\": \"redirect\",\n \"value\": \"Redirect\"\n },\n {\n \"code\": \"insert\",\n \"value\": \"Insert\"\n },\n {\n \"code\": \"hijack\",\n \"value\": \"Hijack\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_user_region\",\n \"label\": \"User Region\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_device_id\",\n \"label\": \"Device ID\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_link_id\",\n \"label\": \"Link ID\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_isp\",\n \"label\": \"ISP\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_device_tag\",\n \"label\": \"Device Tag\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_data_center\",\n \"label\": \"Data Center\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"City A\",\n \"value\": \"City A\"\n },\n {\n \"code\": \"City B\",\n \"value\": \"City B\"\n },\n {\n \"code\": \"City C\",\n \"value\": \"City C\"\n },\n {\n \"code\": \"City D\",\n \"value\": \"City D\"\n },\n {\n \"code\": \"City E\",\n \"value\": \"City E\"\n },\n {\n \"code\": \"City F\",\n \"value\": \"City F\"\n },\n {\n \"code\": \"City G\",\n \"value\": \"City G\"\n },\n {\n \"code\": \"City H\",\n \"value\": \"City H\"\n },\n {\n \"code\": \"City I\",\n \"value\": \"City I\"\n },\n {\n \"code\": \"City J\",\n \"value\": \"City J\"\n },\n {\n \"code\": \"City K\",\n \"value\": \"City K\"\n },\n {\n \"code\": \"City L\",\n \"value\": \"City L\"\n },\n {\n \"code\": \"City M\",\n \"value\": \"City M\"\n },\n {\n \"code\": \"City N\",\n \"value\": \"City N\"\n },\n {\n \"code\": \"City O\",\n \"value\": \"City O\"\n },\n {\n \"code\": \"City P\",\n \"value\": \"City P\"\n },\n {\n \"code\": \"City Q\",\n \"value\": \"City Q\"\n },\n {\n \"code\": \"City R\",\n \"value\": \"City R\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_encapsulation\",\n \"label\": \"Encapsulation\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"Ethernet\"\n },\n {\n \"code\": \"8\",\n \"value\": \"PPP\"\n },\n {\n \"code\": \"12\",\n \"value\": \"CiscoHDLC\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_app_label\",\n \"label\": \"Application Label\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_tunnels\",\n \"label\": \"Tunnels\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_protocol_label\",\n \"label\": \"Protocol Label\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_app_id\",\n \"label\": \"Application ID\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_app_surrogate_id\",\n \"label\": \"Surrogate ID\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_l7_protocol\",\n \"label\": \"L7 Protocol\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_service_category\",\n \"label\": \"FQDN Category\",\n \"doc\": {\n \"visibility\": \"disabled\",\n \"dict_location\": {\n \"path\": \"/v1/category/dict\",\n \"key\": \"categoryId\",\n \"value\": \"categoryName\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_start_time\",\n \"label\": \"Start Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_end_time\",\n \"label\": \"End Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"get_value\",\n \"appendTo\": \"common_recv_time\"\n },\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_establish_latency_ms\",\n \"label\": \"Establish Latency(ms)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_con_duration_ms\",\n \"label\": \"Duration(ms)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_stream_dir\",\n \"label\": \"Stream Direction\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"c2s\"\n },\n {\n \"code\": \"2\",\n \"value\": \"s2c\"\n },\n {\n \"code\": \"3\",\n \"value\": \"double\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_address_list\",\n \"label\": \"Address List\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_has_dup_traffic\",\n \"label\": \"Duplication Traffic\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"No\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Yes\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_stream_error\",\n \"label\": \"Stream Error\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_stream_trace_id\",\n \"label\": \"Session ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_link_info_c2s\",\n \"label\": \"Link Info(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_link_info_s2c\",\n \"label\": \"Link Info(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_c2s_ipfrag_num\",\n \"label\": \"Fragmentation Packets(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_ipfrag_num\",\n \"label\": \"Fragmentation Packets(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_tcp_lostlen\",\n \"label\": \"Sequence Gap Loss(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_tcp_lostlen\",\n \"label\": \"Sequence Gap Loss(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_tcp_unorder_num\",\n \"label\": \"Unorder Packets(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_tcp_unorder_num\",\n \"label\": \"Unorder Packets(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_retrans\",\n \"label\": \"Packet Retransmission(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_retrans\",\n \"label\": \"Packet Retransmission(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_retrans\",\n \"label\": \"Byte Retransmission(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_retrans\",\n \"label\": \"Byte Retransmission(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_tcp_client_isn\",\n \"label\": \"TCP Client ISN\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_tcp_server_isn\",\n \"label\": \"TCP Server ISN\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_first_ttl\",\n \"label\": \"First TTL\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_processing_time\",\n \"label\": \"Processing Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"current_timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"radius_packet_type\",\n \"label\": \"Packet Type\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"Access-Request\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Access-Accept\"\n },\n {\n \"code\": \"3\",\n \"value\": \"Access-Reject\"\n },\n {\n \"code\": \"4\",\n \"value\": \"Accounting-Request\"\n },\n {\n \"code\": \"5\",\n \"value\": \"Accounting-Response\"\n },\n {\n \"code\": \"11\",\n \"value\": \"Access-Challenge\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"radius_account\",\n \"label\": \"Account\",\n \"doc\": {\n \"format\": {\n \"functions\": \"get_value\",\n \"appendTo\": \"common_subscriber_id\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"radius_nas_ip\",\n \"label\": \"Nas IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"radius_framed_ip\",\n \"label\": \"Framed IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"radius_session_timeout\",\n \"label\": \"Session Timeout\",\n \"type\": \"int\"\n },\n {\n \"name\": \"radius_idle_timeout\",\n \"label\": \"Idle Timeout\",\n \"type\": \"int\"\n },\n {\n \"name\": \"radius_acct_status_type\",\n \"label\": \"ACC Status Type\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"Start\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Stop\"\n },\n {\n \"code\": \"3\",\n \"value\": \"Interim-Update\"\n },\n {\n \"code\": \"7\",\n \"value\": \"Accounting-On\"\n },\n {\n \"code\": \"8\",\n \"value\": \"Accounting-Off\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"radius_acct_terminate_cause\",\n \"label\": \"Acct Terminate Cause\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"User Request\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Lost Carrier\"\n },\n {\n \"code\": \"3\",\n \"value\": \"Lost Service\"\n },\n {\n \"code\": \"4\",\n \"value\": \"Idle Timeout\"\n },\n {\n \"code\": \"5\",\n \"value\": \"Session Timeout\"\n },\n {\n \"code\": \"6\",\n \"value\": \"Admin Reset\"\n },\n {\n \"code\": \"7\",\n \"value\": \"Admin Reboot\"\n },\n {\n \"code\": \"8\",\n \"value\": \"Port Error\"\n },\n {\n \"code\": \"9\",\n \"value\": \"NAS Error\"\n },\n {\n \"code\": \"10\",\n \"value\": \"NAS Request\"\n },\n {\n \"code\": \"11\",\n \"value\": \"NAS Reboot\"\n },\n {\n \"code\": \"12\",\n \"value\": \"Port Unneeded\"\n },\n {\n \"code\": \"13\",\n \"value\": \"Port Preempted\"\n },\n {\n \"code\": \"14\",\n \"value\": \"Port Suspended\"\n },\n {\n \"code\": \"15\",\n \"value\": \"Service Unavailable\"\n },\n {\n \"code\": \"16\",\n \"value\": \"Callback\"\n },\n {\n \"code\": \"17\",\n \"value\": \"User Error\"\n },\n {\n \"code\": \"18\",\n \"value\": \"Host Request\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"radius_event_timestamp\",\n \"label\": \"Event Timestamp\",\n \"type\": \"int\"\n },\n {\n \"name\": \"radius_service_type\",\n \"label\": \"Service Type\",\n \"type\": \"int\"\n },\n {\n \"name\": \"radius_nas_port\",\n \"label\": \"Nas Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"radius_framed_protocol\",\n \"label\": \"Framed Protocol\",\n \"type\": \"int\"\n },\n {\n \"name\": \"radius_callback_number\",\n \"label\": \"Callback Number\",\n \"type\": \"string\"\n },\n {\n \"name\": \"radius_callback_id\",\n \"label\": \"Callback ID\",\n \"type\": \"string\"\n },\n {\n \"name\": \"radius_termination_action\",\n \"label\": \"Termination Action\",\n \"type\": \"int\"\n },\n {\n \"name\": \"radius_called_station_id\",\n \"label\": \"Called Station ID\",\n \"type\": \"string\"\n },\n {\n \"name\": \"radius_calling_station_id\",\n \"label\": \"Calling Station ID\",\n \"type\": \"string\"\n },\n {\n \"name\": \"radius_acct_delay_time\",\n \"label\": \"Acct Delay Time\",\n \"type\": \"int\"\n },\n {\n \"name\": \"radius_acct_session_id\",\n \"label\": \"Acct Session ID\",\n \"type\": \"string\"\n },\n {\n \"name\": \"radius_acct_multi_session_id\",\n \"label\": \"Acct Multi Session ID\",\n \"type\": \"string\"\n },\n {\n \"name\": \"radius_acct_input_octets\",\n \"label\": \"Acct Input Octets\",\n \"type\": \"long\"\n },\n {\n \"name\": \"radius_acct_output_octets\",\n \"label\": \"Acct Output Octets\",\n \"type\": \"long\"\n },\n {\n \"name\": \"radius_acct_input_packets\",\n \"label\": \"Acct Input Packets\",\n \"type\": \"long\"\n },\n {\n \"name\": \"radius_acct_output_packets\",\n \"label\": \"Acct Output Packets\",\n \"type\": \"long\"\n },\n {\n \"name\": \"radius_acct_session_time\",\n \"label\": \"Acct Session Time\",\n \"type\": \"int\"\n },\n {\n \"name\": \"radius_acct_link_count\",\n \"label\": \"Acct Link Count\",\n \"type\": \"int\"\n },\n {\n \"name\": \"radius_acct_interim_interval\",\n \"label\": \"Acct Interim Interval\",\n \"type\": \"int\"\n }\n ]\n}', '7a35744a250dbfbc81c4947c16bb4222', '2021-06-01 21:56:52', '2021-06-01 21:56:52', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (30, 'security_event_hits_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"security_event_hits_log\",\n \"namespace\": \"druid\",\n \"doc\": {\n \"partition_key\": \"__time\"\n },\n \"fields\": [\n {\n \"name\": \"__time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"action\",\n \"type\": \"long\"\n },\n {\n \"name\": \"entrance_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"hits\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"s2c_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"isp\",\n \"type\": \"string\"\n },\n {\n \"name\": \"policy_id\",\n \"type\": \"long\"\n }\n ]\n}', '57967b3f7d48c1765ae180645df6812e', '2021-06-01 21:56:53', '2021-06-01 21:56:53', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (31, 'security_event_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"security_event_log\",\n \"namespace\": \"tsg_galaxy_v3\",\n \"doc\": {\n \"primary_key\": \"common_log_id\",\n \"partition_key\": \"common_recv_time\",\n \"functions\": {\n \"aggregation\": [\n {\n \"name\": \"COUNT\",\n \"label\": \"COUNT\",\n \"function\": \"count(expr)\"\n },\n {\n \"name\": \"COUNT_DISTINCT\",\n \"label\": \"COUNT_DISTINCT\",\n \"function\": \"count(distinct expr)\"\n },\n {\n \"name\": \"AVG\",\n \"label\": \"AVG\",\n \"function\": \"avg(expr)\"\n },\n {\n \"name\": \"SUM\",\n \"label\": \"SUM\",\n \"function\": \"sum(expr)\"\n },\n {\n \"name\": \"MAX\",\n \"label\": \"MAX\",\n \"function\": \"max(expr)\"\n },\n {\n \"name\": \"MIN\",\n \"label\": \"MIN\",\n \"function\": \"min(expr)\"\n }\n ],\n \"operator\": [\n {\n \"name\": \"=\",\n \"label\": \"=\",\n \"function\": \"expr = value\"\n },\n {\n \"name\": \"!=\",\n \"label\": \"!=\",\n \"function\": \"expr != value\"\n },\n {\n \"name\": \">\",\n \"label\": \">\",\n \"function\": \"expr > value\"\n },\n {\n \"name\": \"<\",\n \"label\": \"<\",\n \"function\": \"expr < value\"\n },\n {\n \"name\": \">=\",\n \"label\": \">=\",\n \"function\": \"expr >= value\"\n },\n {\n \"name\": \"<=\",\n \"label\": \"<=\",\n \"function\": \"expr <= value\"\n },\n {\n \"name\": \"in\",\n \"label\": \"IN\",\n \"function\": \"expr in (values)\"\n },\n {\n \"name\": \"not in\",\n \"label\": \"NOT IN\",\n \"function\": \"expr not in (values)\"\n },\n {\n \"name\": \"like\",\n \"label\": \"LIKE\",\n \"function\": \"expr like value\"\n },\n {\n \"name\": \"not like\",\n \"label\": \"NOT LIKE\",\n \"function\": \"expr not like value\"\n },\n {\n \"name\": \"notEmpty\",\n \"label\": \"NOT EMPTY\",\n \"function\": \"notEmpty(expr)\"\n },\n {\n \"name\": \"empty\",\n \"label\": \"EMPTY\",\n \"function\": \"empty(expr)\"\n }\n ]\n },\n \"schema_query\": {\n \"dimensions\": [\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_policy_id\",\n \"common_action\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_client_location\",\n \"common_server_location\",\n \"common_subscriber_id\",\n \"common_client_port\",\n \"common_server_port\",\n \"common_schema_type\",\n \"common_l4_protocol\",\n \"common_l7_protocol\",\n \"common_data_center\",\n \"common_client_asn\",\n \"common_server_asn\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"http_host\",\n \"http_domain\",\n \"http_url\",\n \"ssl_sni\",\n \"ssl_ja3_hash\",\n \"ssl_client_side_version\",\n \"ssl_server_side_version\",\n \"mail_account\",\n \"mail_from\",\n \"mail_to\",\n \"quic_sni\"\n ],\n \"metrics\": [\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_subscriber_id\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"http_host\",\n \"http_domain\",\n \"http_url\",\n \"ssl_sni\",\n \"ssl_ja3_hash\",\n \"ssl_client_side_latency\",\n \"ssl_server_side_latency\",\n \"mail_account\",\n \"mail_from\",\n \"mail_to\",\n \"quic_sni\"\n ],\n \"filters\": [\n \"common_policy_id\",\n \"common_action\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_client_port\",\n \"common_server_port\",\n \"common_client_location\",\n \"common_server_location\",\n \"common_subscriber_id\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_l4_protocol\",\n \"common_l7_protocol\",\n \"common_stream_dir\",\n \"common_data_center\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_direction\",\n \"common_schema_type\",\n \"common_client_asn\",\n \"common_server_asn\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"http_host\",\n \"http_domain\",\n \"http_url\",\n \"http_content_type\",\n \"ssl_sni\",\n \"ssl_ja3_hash\",\n \"ssl_pinningst\",\n \"ssl_intercept_state\",\n \"ssl_client_side_version\",\n \"ssl_server_side_version\",\n \"ssl_cert_verify\",\n \"ssl_client_side_latency\",\n \"ssl_server_side_latency\",\n \"mail_account\",\n \"mail_from\",\n \"mail_to\",\n \"mail_subject\",\n \"quic_sni\"\n ],\n \"references\": {\n \"aggregation\": [\n {\n \"type\": \"int\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"long\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"float\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"double\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"string\",\n \"functions\": \"COUNT,COUNT_DISTINCT\"\n },\n {\n \"type\": \"date\",\n \"functions\": \"COUNT,COUNT_DISTINCT,MAX,MIN\"\n },\n {\n \"type\": \"timestamp\",\n \"functions\": \"COUNT,COUNT_DISTINCT,MAX,MIN\"\n }\n ],\n \"operator\": [\n {\n \"type\": \"int\",\n \"functions\": \"=,!=,>,<,>=,<=,in,not in\"\n },\n {\n \"type\": \"long\",\n \"functions\": \"=,!=,>,<,>=,<=,in,not in\"\n },\n {\n \"type\": \"float\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"double\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"string\",\n \"functions\": \"=,!=,in,not in,like,not like,notEmpty,empty\"\n },\n {\n \"type\": \"date\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"timestamp\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n }\n ]\n },\n \"details\": {\n \"general\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_stream_trace_id\",\n \"common_direction\",\n \"common_stream_dir\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_processing_time\",\n \"common_entrance_id\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_data_center\",\n \"common_sled_ip\"\n ],\n \"action\": [\n \"common_action\",\n \"common_sub_action\",\n \"common_policy_id\",\n \"common_user_tags\",\n \"common_user_region\"\n ],\n \"source\": [\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_client_port\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\"\n ],\n \"destination\": [\n \"common_server_ip\",\n \"common_external_ip\",\n \"common_server_port\",\n \"common_server_location\",\n \"common_server_asn\"\n ],\n \"application\": [\n \"common_app_id\",\n \"common_app_label\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_protocol_label\",\n \"common_service_category\",\n \"common_service\",\n \"common_l4_protocol\"\n ],\n \"transmission\": [\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_first_ttl\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\"\n ],\n \"other\": [\n \"common_address_type\",\n \"common_schema_type\",\n \"common_device_tag\",\n \"common_encapsulation\",\n \"common_tunnels\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\"\n ]\n }\n },\n \"schema_type\": {\n \"BASE\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"HTTP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"http_url\",\n \"http_host\",\n \"http_domain\",\n \"http_request_line\",\n \"http_response_line\",\n \"http_request_header\",\n \"http_response_header\",\n \"http_request_content\",\n \"http_response_content\",\n \"http_request_body\",\n \"http_response_body\",\n \"http_request_body_key\",\n \"http_response_body_key\",\n \"http_proxy_flag\",\n \"http_sequence\",\n \"http_snapshot\",\n \"http_cookie\",\n \"http_referer\",\n \"http_user_agent\",\n \"http_content_length\",\n \"http_content_type\",\n \"http_set_cookie\",\n \"http_version\",\n \"http_response_lantency_ms\",\n \"http_session_duration_ms\",\n \"http_action_file_size\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"http_url\",\n \"common_server_port\"\n ]\n },\n \"MAIL\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"mail_protocol_type\",\n \"mail_account\",\n \"mail_from_cmd\",\n \"mail_to_cmd\",\n \"mail_from\",\n \"mail_to\",\n \"mail_cc\",\n \"mail_bcc\",\n \"mail_subject\",\n \"mail_subject_charset\",\n \"mail_content\",\n \"mail_content_charset\",\n \"mail_attachment_name\",\n \"mail_attachment_name_charset\",\n \"mail_attachment_content\",\n \"mail_eml_file\",\n \"mail_snapshot\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"mail_from\",\n \"mail_to\",\n \"mail_subject\"\n ]\n },\n \"DNS\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"dns_message_id\",\n \"dns_qr\",\n \"dns_opcode\",\n \"dns_aa\",\n \"dns_tc\",\n \"dns_rd\",\n \"dns_ra\",\n \"dns_rcode\",\n \"dns_qdcount\",\n \"dns_ancount\",\n \"dns_nscount\",\n \"dns_arcount\",\n \"dns_qname\",\n \"dns_qtype\",\n \"dns_qclass\",\n \"dns_cname\",\n \"dns_sub\",\n \"dns_rr\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_client_ip\",\n \"dns_qr\",\n \"dns_qname\",\n \"dns_qtype\"\n ]\n },\n \"SSL\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"ssl_sni\",\n \"ssl_san\",\n \"ssl_cn\",\n \"ssl_pinningst\",\n \"ssl_intercept_state\",\n \"ssl_server_side_latency\",\n \"ssl_client_side_latency\",\n \"ssl_server_side_version\",\n \"ssl_client_side_version\",\n \"ssl_cert_verify\",\n \"ssl_error\",\n \"ssl_con_latency_ms\",\n \"ssl_ja3_fingerprint\",\n \"ssl_ja3_hash\",\n \"ssl_cert_issuer\",\n \"ssl_cert_subject\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"ssl_sni\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"QUIC\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"quic_version\",\n \"quic_sni\",\n \"quic_user_agent\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"quic_sni\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"FTP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"ftp_account\",\n \"ftp_url\",\n \"ftp_content\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"ftp_url\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"BGP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"bgp_type\",\n \"bgp_as_num\",\n \"bgp_route\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"bgp_type\",\n \"bgp_as_num\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"SIP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"sip_call_id\",\n \"sip_originator_description\",\n \"sip_responder_description\",\n \"sip_user_agent\",\n \"sip_server\",\n \"sip_originator_sdp_connect_ip\",\n \"sip_originator_sdp_media_port\",\n \"sip_originator_sdp_media_type\",\n \"sip_originator_sdp_content\",\n \"sip_responder_sdp_connect_ip\",\n \"sip_responder_sdp_media_port\",\n \"sip_responder_sdp_media_type\",\n \"sip_responder_sdp_content\",\n \"sip_duration\",\n \"sip_bye\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"sip_originator_description\",\n \"sip_responder_description\",\n \"sip_call_id\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"RTP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"rtp_payload_type_c2s\",\n \"rtp_payload_type_s2c\",\n \"rtp_pcap_path\",\n \"rtp_originator_dir\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\",\n \"rtp_pcap_path\"\n ]\n },\n \"APP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"app_extra_info\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_app_id\",\n \"common_app_label\",\n \"app_extra_info\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n }\n },\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_schema_type\"\n ],\n \"tunnel_type\": {\n \"GTP\": [\n {\n \"name\": \"gtp_sgw_ip\",\n \"label\": \"S-GW IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_pgw_ip\",\n \"label\": \"P-GW IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_sgw_port\",\n \"label\": \"S-GW Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"gtp_pgw_port\",\n \"label\": \"P-GW Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"gtp_uplink_teid\",\n \"label\": \"Uplink TEID\",\n \"type\": \"long\"\n },\n {\n \"name\": \"gtp_downlink_teid\",\n \"label\": \"Downlink TEID\",\n \"type\": \"long\"\n }\n ],\n \"MPLS\": [\n {\n \"name\": \"mpls_c2s_direction_label\",\n \"label\": \"Multiprotocol Label(c2s)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n },\n {\n \"name\": \"mpls_s2c_direction_label\",\n \"label\": \"Multiprotocol Label(s2c)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n }\n ],\n \"VLAN\": [\n {\n \"name\": \"vlan_c2s_direction_id\",\n \"label\": \"VLAN Direction(c2s)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n },\n {\n \"name\": \"vlan_s2c_direction_id\",\n \"label\": \"VLAN Direction(s2c)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n }\n ],\n \"ETHERNET\": [\n {\n \"name\": \"source_mac\",\n \"label\": \"Source MAC\",\n \"type\": \"string\"\n },\n {\n \"name\": \"destination_mac\",\n \"label\": \"Destination MAC\",\n \"type\": \"string\"\n }\n ],\n \"MULTIPATH_ETHERNET\": [\n {\n \"name\": \"c2s_source_mac\",\n \"label\": \"Source MAC(c2s)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"c2s_destination_mac\",\n \"label\": \"Destination MAC(c2s)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_source_mac\",\n \"label\": \"Source MAC(s2c)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_destination_mac\",\n \"label\": \"Destination MAC(s2c)\",\n \"type\": \"string\"\n }\n ]\n }\n },\n \"fields\": [\n {\n \"name\": \"common_recv_time\",\n \"label\": \"Receive Time\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_log_id\",\n \"label\": \"Log ID\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"snowflake_id\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_policy_id\",\n \"label\": \"Policy ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_subscriber_id\",\n \"label\": \"Subscriber ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_imei\",\n \"label\": \"IMEI\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_imsi\",\n \"label\": \"IMSI\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_phone_number\",\n \"label\": \"Phone Number\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_ip\",\n \"label\": \"Client IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"geo_asn,radius_match\",\n \"appendTo\": \"common_client_asn,common_subscriber_id\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_internal_ip\",\n \"label\": \"Internal IP\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"if\",\n \"param\": \"$.common_direction=69,$.common_client_ip,$.common_server_ip\"\n },\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_port\",\n \"label\": \"Client Port\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_l4_protocol\",\n \"label\": \"L4 Protocol\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_address_type\",\n \"label\": \"Address Type\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"4\",\n \"value\": \"ipv4\"\n },\n {\n \"code\": \"6\",\n \"value\": \"ipv6\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_server_ip\",\n \"label\": \"Server IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"geo_asn\",\n \"appendTo\": \"common_server_asn\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_port\",\n \"label\": \"Server Port\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_external_ip\",\n \"label\": \"External IP\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"if\",\n \"param\": \"$.common_direction=73,$.common_client_ip,$.common_server_ip\"\n },\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_action\",\n \"label\": \"Action\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Intercept\"\n },\n {\n \"code\": \"16\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"128\",\n \"value\": \"Allow\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_direction\",\n \"label\": \"Direction\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"69\",\n \"value\": \"outbound\"\n },\n {\n \"code\": \"73\",\n \"value\": \"inbound\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_entrance_id\",\n \"label\": \"Entrance ID\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_sled_ip\",\n \"label\": \"Sled IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_location\",\n \"label\": \"Client Location\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_asn\",\n \"label\": \"Client ASN\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_location\",\n \"label\": \"Server Location\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_asn\",\n \"label\": \"Server ASN\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sessions\",\n \"label\": \"Sessions\",\n \"doc\": {\n \"format\": {\n \"functions\": \"set_value\",\n \"param\": \"1\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_num\",\n \"label\": \"Packets Sent\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_num\",\n \"label\": \"Packets Received\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_num\",\n \"label\": \"Bytes Sent\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_num\",\n \"label\": \"Bytes Received\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_diff\",\n \"label\": \"Packets Sent(Diff)\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_diff\",\n \"label\": \"Packets Received(Diff)\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_diff\",\n \"label\": \"Bytes Sent(Diff)\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_diff\",\n \"label\": \"Bytes Received(Diff)\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_service\",\n \"label\": \"Service\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_schema_type\",\n \"label\": \"Schema Type\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"HTTP\",\n \"value\": \"HTTP\"\n },\n {\n \"code\": \"MAIL\",\n \"value\": \"MAIL\"\n },\n {\n \"code\": \"DNS\",\n \"value\": \"DNS\"\n },\n {\n \"code\": \"SSL\",\n \"value\": \"SSL\"\n },\n {\n \"code\": \"QUIC\",\n \"value\": \"QUIC\"\n },\n {\n \"code\": \"FTP\",\n \"value\": \"FTP\"\n },\n {\n \"code\": \"SIP\",\n \"value\": \"SIP\"\n },\n {\n \"code\": \"RTP\",\n \"value\": \"RTP\"\n },\n {\n \"code\": \"APP\",\n \"value\": \"APP\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_user_tags\",\n \"label\": \"User Tags\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sub_action\",\n \"label\": \"Sub Action\",\n \"doc\": {\n \"data\": [\n {\n \"code\": \"allow\",\n \"value\": \"Allow\"\n },\n {\n \"code\": \"deny\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"monitor\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"replace\",\n \"value\": \"Replace\"\n },\n {\n \"code\": \"redirect\",\n \"value\": \"Redirect\"\n },\n {\n \"code\": \"insert\",\n \"value\": \"Insert\"\n },\n {\n \"code\": \"hijack\",\n \"value\": \"Hijack\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_user_region\",\n \"label\": \"User Region\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_device_id\",\n \"label\": \"Device ID\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_link_id\",\n \"label\": \"Link ID\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_isp\",\n \"label\": \"ISP\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_device_tag\",\n \"label\": \"Device Tag\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_data_center\",\n \"label\": \"Data Center\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"City A\",\n \"value\": \"City A\"\n },\n {\n \"code\": \"City B\",\n \"value\": \"City B\"\n },\n {\n \"code\": \"City C\",\n \"value\": \"City C\"\n },\n {\n \"code\": \"City D\",\n \"value\": \"City D\"\n },\n {\n \"code\": \"City E\",\n \"value\": \"City E\"\n },\n {\n \"code\": \"City F\",\n \"value\": \"City F\"\n },\n {\n \"code\": \"City G\",\n \"value\": \"City G\"\n },\n {\n \"code\": \"City H\",\n \"value\": \"City H\"\n },\n {\n \"code\": \"City I\",\n \"value\": \"City I\"\n },\n {\n \"code\": \"City J\",\n \"value\": \"City J\"\n },\n {\n \"code\": \"City K\",\n \"value\": \"City K\"\n },\n {\n \"code\": \"City L\",\n \"value\": \"City L\"\n },\n {\n \"code\": \"City M\",\n \"value\": \"City M\"\n },\n {\n \"code\": \"City N\",\n \"value\": \"City N\"\n },\n {\n \"code\": \"City O\",\n \"value\": \"City O\"\n },\n {\n \"code\": \"City P\",\n \"value\": \"City P\"\n },\n {\n \"code\": \"City Q\",\n \"value\": \"City Q\"\n },\n {\n \"code\": \"City R\",\n \"value\": \"City R\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_encapsulation\",\n \"label\": \"Encapsulation\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"Ethernet\"\n },\n {\n \"code\": \"8\",\n \"value\": \"PPP\"\n },\n {\n \"code\": \"12\",\n \"value\": \"CiscoHDLC\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_app_label\",\n \"label\": \"Application Label\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_tunnels\",\n \"label\": \"Tunnels\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_protocol_label\",\n \"label\": \"Protocol Label\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_app_id\",\n \"label\": \"Application ID\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_app_surrogate_id\",\n \"label\": \"Surrogate ID\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_l7_protocol\",\n \"label\": \"L7 Protocol\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_service_category\",\n \"label\": \"FQDN Category\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"dict_location\": {\n \"path\": \"/v1/category/dict\",\n \"key\": \"categoryId\",\n \"value\": \"categoryName\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_start_time\",\n \"label\": \"Start Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_end_time\",\n \"label\": \"End Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"get_value\",\n \"appendTo\": \"common_recv_time\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_establish_latency_ms\",\n \"label\": \"Establish Latency(ms)\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_con_duration_ms\",\n \"label\": \"Duration(ms)\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_stream_dir\",\n \"label\": \"Stream Direction\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"c2s\"\n },\n {\n \"code\": \"2\",\n \"value\": \"s2c\"\n },\n {\n \"code\": \"3\",\n \"value\": \"double\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_address_list\",\n \"label\": \"Address List\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_has_dup_traffic\",\n \"label\": \"Duplication Traffic\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"No\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Yes\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_stream_error\",\n \"label\": \"Stream Error\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_stream_trace_id\",\n \"label\": \"Session ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_link_info_c2s\",\n \"label\": \"Link Info(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_link_info_s2c\",\n \"label\": \"Link Info(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_c2s_ipfrag_num\",\n \"label\": \"Fragmentation Packets(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_ipfrag_num\",\n \"label\": \"Fragmentation Packets(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_tcp_lostlen\",\n \"label\": \"Sequence Gap Loss(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_tcp_lostlen\",\n \"label\": \"Sequence Gap Loss(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_tcp_unorder_num\",\n \"label\": \"Unorder Packets(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_tcp_unorder_num\",\n \"label\": \"Unorder Packets(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_retrans\",\n \"label\": \"Packet Retransmission(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_retrans\",\n \"label\": \"Packet Retransmission(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_retrans\",\n \"label\": \"Byte Retransmission(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_retrans\",\n \"label\": \"Byte Retransmission(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_tcp_client_isn\",\n \"label\": \"TCP Client ISN\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_tcp_server_isn\",\n \"label\": \"TCP Server ISN\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_first_ttl\",\n \"label\": \"First TTL\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_processing_time\",\n \"label\": \"Processing Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"current_timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"http_url\",\n \"label\": \"HTTP.URL\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_host\",\n \"label\": \"HTTP.Host\",\n \"doc\": {\n \"format\": {\n \"functions\": \"sub_domain\",\n \"appendTo\": \"http_domain\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_domain\",\n \"label\": \"HTTP.Domain\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_line\",\n \"label\": \"HTTP.Request Line\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_line\",\n \"label\": \"HTTP.Response Line\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_header\",\n \"label\": \"HTTP.Request Header\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_header\",\n \"label\": \"HTTP.Response Header\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_content\",\n \"label\": \"HTTP.Request Content\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_content\",\n \"label\": \"HTTP.Response Content\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_body\",\n \"label\": \"HTTP.Request Body\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"file\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_body\",\n \"label\": \"HTTP.Response Body\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"file\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_request_body_key\",\n \"label\": \"HTTP.Request Body Key\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_body_key\",\n \"label\": \"HTTP.Response Body Key\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_proxy_flag\",\n \"label\": \"HTTP.Proxy Flag\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"http_sequence\",\n \"label\": \"HTTP.Sequence\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"http_snapshot\",\n \"label\": \"HTTP.Snapshot\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"http_cookie\",\n \"label\": \"HTTP.Cookie\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_referer\",\n \"label\": \"HTTP.Referer\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_user_agent\",\n \"label\": \"HTTP.User Agent\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_content_length\",\n \"label\": \"HTTP.Content Length\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_content_type\",\n \"label\": \"HTTP.Content Type\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_set_cookie\",\n \"label\": \"HTTP.Set Cookie\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_version\",\n \"label\": \"HTTP.Version\",\n \"type\": \"string\"\n },\n {\n \"name\": \"http_response_lantency_ms\",\n \"label\": \"HTTP.Response Latency(ms)\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"http_action_file_size\",\n \"label\": \"HTTP.Action File Size\",\n \"type\": \"int\"\n },\n {\n \"name\": \"http_session_duration_ms\",\n \"label\": \"HTTP.Session Duration(ms)\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"mail_protocol_type\",\n \"label\": \"Mail.Protocol Type\",\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_account\",\n \"label\": \"Mail.Account\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_from_cmd\",\n \"label\": \"Mail.From CMD\",\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_to_cmd\",\n \"label\": \"Mail.To CMD\",\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_from\",\n \"label\": \"Mail.From\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"email\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_to\",\n \"label\": \"Mail.To\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"email\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_cc\",\n \"label\": \"Mail.CC\",\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_bcc\",\n \"label\": \"Mail.BCC\",\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_subject\",\n \"label\": \"Mail.Subject\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"decode_of_base64\",\n \"param\": \"$.mail_subject_charset\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_subject_charset\",\n \"label\": \"Mail.Subject Charset\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_content\",\n \"label\": \"Mail.Content\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_content_charset\",\n \"label\": \"Mail.Content Charset\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_attachment_name\",\n \"label\": \"Mail.Attachment\",\n \"doc\": {\n \"format\": {\n \"functions\": \"decode_of_base64\",\n \"param\": \"$.mail_attachment_name_charset\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_attachment_name_charset\",\n \"label\": \"Mail.Attachment Charset\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_attachment_content\",\n \"label\": \"Mail.Attachment Content\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_eml_file\",\n \"label\": \"Mail.EML File\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"file\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"mail_snapshot\",\n \"label\": \"Mail.Snapshot\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"dns_message_id\",\n \"label\": \"DNS.Message ID\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_qr\",\n \"label\": \"DNS.QR\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"QUERY\"\n },\n {\n \"code\": \"1\",\n \"value\": \"REESPONSE\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_opcode\",\n \"label\": \"DNS.OPCODE\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"QUERY\"\n },\n {\n \"code\": \"1\",\n \"value\": \"IQUERY\"\n },\n {\n \"code\": \"2\",\n \"value\": \"STATUS\"\n },\n {\n \"code\": \"5\",\n \"value\": \"UPDATE\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_aa\",\n \"label\": \"DNS.AA\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_tc\",\n \"label\": \"DNS.TC\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_rd\",\n \"label\": \"DNS.RD\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_ra\",\n \"label\": \"DNS.RA\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_rcode\",\n \"label\": \"DNS.RCODE\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_qdcount\",\n \"label\": \"DNS.QDCOUNT\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_ancount\",\n \"label\": \"DNS.ANCOUNT\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_nscount\",\n \"label\": \"DNS.NSCOUNT\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_arcount\",\n \"label\": \"DNS.ARCOUNT\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_qname\",\n \"label\": \"DNS.QNAME\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"dns_qtype\",\n \"label\": \"DNS.QTYPE\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"A\"\n },\n {\n \"code\": \"2\",\n \"value\": \"NS\"\n },\n {\n \"code\": \"5\",\n \"value\": \"CNAME\"\n },\n {\n \"code\": \"6\",\n \"value\": \"SOA\"\n },\n {\n \"code\": \"11\",\n \"value\": \"WKS\"\n },\n {\n \"code\": \"12\",\n \"value\": \"PTR\"\n },\n {\n \"code\": \"13\",\n \"value\": \"HINFO\"\n },\n {\n \"code\": \"11\",\n \"value\": \"WKS\"\n },\n {\n \"code\": \"15\",\n \"value\": \"MX\"\n },\n {\n \"code\": \"28\",\n \"value\": \"AAAA\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_qclass\",\n \"label\": \"DNS.QCLASS\",\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_cname\",\n \"label\": \"DNS.CNAME\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"dns_sub\",\n \"label\": \"DNS.SUB\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"DNS\"\n },\n {\n \"code\": \"2\",\n \"value\": \"DNSSEC\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"dns_rr\",\n \"label\": \"DNS.RR\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_version\",\n \"label\": \"SSL.Version\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_sni\",\n \"label\": \"SSL.SNI\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"sub_domain\",\n \"appendTo\": \"http_domain\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_san\",\n \"label\": \"SSL.SAN\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_cn\",\n \"label\": \"SSL.CN\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_pinningst\",\n \"label\": \"SSL.Pinning\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"Not Pinning\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Pinning\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Maybe Pinning\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_intercept_state\",\n \"label\": \"SSL.Intercept State\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"Passthrough\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Intercept\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Shutdown\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_server_side_latency\",\n \"label\": \"SSL.Server Side Latency(ms)\",\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_client_side_latency\",\n \"label\": \"SSL.Client Side Latency(ms)\",\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_server_side_version\",\n \"label\": \"SSL.Server Side Version\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_client_side_version\",\n \"label\": \"SSL.Client Side Version\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_cert_verify\",\n \"label\": \"SSL.Certificate Verify\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"No\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Yes\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_error\",\n \"label\": \"SSL.Error\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_con_latency_ms\",\n \"label\": \"SSL.Connection Latency(ms)\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"ssl_ja3_fingerprint\",\n \"label\": \"SSL.JA3\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_ja3_hash\",\n \"label\": \"SSL.JA3 hash\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_cert_issuer\",\n \"label\": \"SSL.Issuer\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ssl_cert_subject\",\n \"label\": \"SSL.Subject\",\n \"type\": \"string\"\n },\n {\n \"name\": \"quic_version\",\n \"label\": \"Quic.Version\",\n \"type\": \"string\"\n },\n {\n \"name\": \"quic_sni\",\n \"label\": \"Quic.SNI\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"sub_domain\",\n \"appendTo\": \"http_domain\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"quic_user_agent\",\n \"label\": \"Quic.User Agent\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ftp_account\",\n \"label\": \"FTP.Account\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"ftp_url\",\n \"label\": \"FTP.URL\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ftp_content\",\n \"label\": \"FTP.Content\",\n \"type\": \"string\"\n },\n {\n \"name\": \"bgp_type\",\n \"label\": \"BGP.Type\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"bgp_as_num\",\n \"label\": \"BGP.AS Number\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"bgp_route\",\n \"label\": \"BGP.Route\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"voip_calling_account\",\n \"label\": \"VoIP.Calling Account\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"voip_called_account\",\n \"label\": \"VoIP.Called Account\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"voip_calling_number\",\n \"label\": \"VoIP.Calling Number\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"voip_called_number\",\n \"label\": \"VoIP.Called Number\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"streaming_media_url\",\n \"label\": \"Streaming.Media URL\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"streaming_media_protocol\",\n \"label\": \"Streaming.Media Protocol\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"app_extra_info\",\n \"label\": \"APP.Extra Info\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_call_id\",\n \"label\": \"SIP.Call-ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_originator_description\",\n \"label\": \"SIP.Originator\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_responder_description\",\n \"label\": \"SIP.Responder\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_user_agent\",\n \"label\": \"SIP.User-Agent\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_server\",\n \"label\": \"SIP.Server\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_originator_sdp_connect_ip\",\n \"label\": \"SIP.Originator IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_originator_sdp_media_port\",\n \"label\": \"SIP.Originator Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"sip_originator_sdp_media_type\",\n \"label\": \"SIP.Originator Media Type\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_originator_sdp_content\",\n \"label\": \"SIP.Originator Content\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_responder_sdp_connect_ip\",\n \"label\": \"SIP.Responder IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_responder_sdp_media_port\",\n \"label\": \"SIP.Responder Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"sip_responder_sdp_media_type\",\n \"label\": \"SIP.Responder Media Type\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_responder_sdp_content\",\n \"label\": \"SIP.Responder Content\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_duration\",\n \"label\": \"SIP.Duration\",\n \"type\": \"int\"\n },\n {\n \"name\": \"sip_bye\",\n \"label\": \"SIP.Bye\",\n \"type\": \"string\"\n },\n {\n \"name\": \"rtp_payload_type_c2s\",\n \"label\": \"RTP.Payload Type(c2s)\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"PCMU\"\n },\n {\n \"code\": \"1\",\n \"value\": \"1016\"\n },\n {\n \"code\": \"2\",\n \"value\": \"G721\"\n },\n {\n \"code\": \"3\",\n \"value\": \"GSM\"\n },\n {\n \"code\": \"4\",\n \"value\": \"G723\"\n },\n {\n \"code\": \"5\",\n \"value\": \"DVI4_8000\"\n },\n {\n \"code\": \"6\",\n \"value\": \"DVI4_16000\"\n },\n {\n \"code\": \"7\",\n \"value\": \"LPC\"\n },\n {\n \"code\": \"8\",\n \"value\": \"PCMA\"\n },\n {\n \"code\": \"9\",\n \"value\": \"G722\"\n },\n {\n \"code\": \"10\",\n \"value\": \"L16_STEREO\"\n },\n {\n \"code\": \"11\",\n \"value\": \"L16_MONO\"\n },\n {\n \"code\": \"12\",\n \"value\": \"QCELP\"\n },\n {\n \"code\": \"13\",\n \"value\": \"CN\"\n },\n {\n \"code\": \"14\",\n \"value\": \"MPA\"\n },\n {\n \"code\": \"15\",\n \"value\": \"G728\"\n },\n {\n \"code\": \"16\",\n \"value\": \"DVI4_11025\"\n },\n {\n \"code\": \"17\",\n \"value\": \"DVI4_22050\"\n },\n {\n \"code\": \"18\",\n \"value\": \"G729\"\n },\n {\n \"code\": \"19\",\n \"value\": \"CN_OLD\"\n },\n {\n \"code\": \"25\",\n \"value\": \"CELB\"\n },\n {\n \"code\": \"26\",\n \"value\": \"JPEG\"\n },\n {\n \"code\": \"28\",\n \"value\": \"NV\"\n },\n {\n \"code\": \"31\",\n \"value\": \"H261\"\n },\n {\n \"code\": \"32\",\n \"value\": \"MPV\"\n },\n {\n \"code\": \"33\",\n \"value\": \"MP2T\"\n },\n {\n \"code\": \"34\",\n \"value\": \"H263\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"rtp_payload_type_s2c\",\n \"label\": \"RTP.Payload Type(s2c)\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"PCMU\"\n },\n {\n \"code\": \"1\",\n \"value\": \"1016\"\n },\n {\n \"code\": \"2\",\n \"value\": \"G721\"\n },\n {\n \"code\": \"3\",\n \"value\": \"GSM\"\n },\n {\n \"code\": \"4\",\n \"value\": \"G723\"\n },\n {\n \"code\": \"5\",\n \"value\": \"DVI4_8000\"\n },\n {\n \"code\": \"6\",\n \"value\": \"DVI4_16000\"\n },\n {\n \"code\": \"7\",\n \"value\": \"LPC\"\n },\n {\n \"code\": \"8\",\n \"value\": \"PCMA\"\n },\n {\n \"code\": \"9\",\n \"value\": \"G722\"\n },\n {\n \"code\": \"10\",\n \"value\": \"L16_STEREO\"\n },\n {\n \"code\": \"11\",\n \"value\": \"L16_MONO\"\n },\n {\n \"code\": \"12\",\n \"value\": \"QCELP\"\n },\n {\n \"code\": \"13\",\n \"value\": \"CN\"\n },\n {\n \"code\": \"14\",\n \"value\": \"MPA\"\n },\n {\n \"code\": \"15\",\n \"value\": \"G728\"\n },\n {\n \"code\": \"16\",\n \"value\": \"DVI4_11025\"\n },\n {\n \"code\": \"17\",\n \"value\": \"DVI4_22050\"\n },\n {\n \"code\": \"18\",\n \"value\": \"G729\"\n },\n {\n \"code\": \"19\",\n \"value\": \"CN_OLD\"\n },\n {\n \"code\": \"25\",\n \"value\": \"CELB\"\n },\n {\n \"code\": \"26\",\n \"value\": \"JPEG\"\n },\n {\n \"code\": \"28\",\n \"value\": \"NV\"\n },\n {\n \"code\": \"31\",\n \"value\": \"H261\"\n },\n {\n \"code\": \"32\",\n \"value\": \"MPV\"\n },\n {\n \"code\": \"33\",\n \"value\": \"MP2T\"\n },\n {\n \"code\": \"34\",\n \"value\": \"H263\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"rtp_pcap_path\",\n \"label\": \"RTP.PCAP\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"file\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"rtp_originator_dir\",\n \"label\": \"RTP.Direction\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"unknown\"\n },\n {\n \"code\": \"1\",\n \"value\": \"c2s\"\n },\n {\n \"code\": \"2\",\n \"value\": \"s2c\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n }\n ]\n}', '96b76aeea00d6bd0526b64c5253ebf0b', '2021-06-01 21:56:54', '2021-06-01 21:56:54', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (32, 'sys_packet_capture_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"sys_packet_capture_log\",\n \"namespace\": \"tsg_galaxy_v3\",\n \"doc\": {\n \"primary_key\": \"common_log_id\",\n \"partition_key\": \"common_recv_time\"\n },\n \"fields\": [\n {\n \"name\": \"common_recv_time\",\n \"type\": \"long\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"current_timestamp\"\n }\n },\n \"label\": \"Receive Time\"\n },\n {\n \"name\": \"common_log_id\",\n \"type\": \"long\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"snowflake_id\"\n }\n },\n \"label\": \"Log ID\"\n },\n {\n \"name\": \"common_policy_id\",\n \"type\": \"long\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Policy ID\"\n },\n {\n \"name\": \"common_subscriber_id\",\n \"type\": \"string\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"label\": \"Subscriber ID\"\n },\n {\n \"name\": \"common_imei\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"IMEI\"\n },\n {\n \"name\": \"common_imsi\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"IMSI\"\n },\n {\n \"name\": \"common_phone_number\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"Phone Number\"\n },\n {\n \"name\": \"common_client_ip\",\n \"type\": \"string\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n }\n },\n \"label\": \"Client IP\"\n },\n {\n \"name\": \"common_internal_ip\",\n \"type\": \"string\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n }\n },\n \"label\": \"Internal IP\"\n },\n {\n \"name\": \"common_client_port\",\n \"type\": \"int\",\n \"label\": \"Client Port\"\n },\n {\n \"name\": \"common_l4_protocol\",\n \"type\": \"string\",\n \"label\": \"L4 Protocol\"\n },\n {\n \"name\": \"common_address_type\",\n \"type\": \"int\",\n \"doc\": {\n \"data\": [\n {\n \"code\": \"4\",\n \"value\": \"ipv4\"\n },\n {\n \"code\": \"6\",\n \"value\": \"ipv6\"\n }\n ]\n },\n \"label\": \"Address Type\"\n },\n {\n \"name\": \"common_server_ip\",\n \"type\": \"string\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n }\n },\n \"label\": \"Server IP\"\n },\n {\n \"name\": \"common_server_port\",\n \"type\": \"int\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"label\": \"Server Port\"\n },\n {\n \"name\": \"common_external_ip\",\n \"type\": \"string\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n }\n },\n \"label\": \"External IP\"\n },\n {\n \"name\": \"common_action\",\n \"type\": \"int\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"None\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Intercept\"\n },\n {\n \"code\": \"16\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"128\",\n \"value\": \"Allow\"\n }\n ]\n },\n \"label\": \"Action\"\n },\n {\n \"name\": \"common_direction\",\n \"type\": \"int\",\n \"doc\": {\n \"data\": [\n {\n \"code\": \"69\",\n \"value\": \"outbound\"\n },\n {\n \"code\": \"73\",\n \"value\": \"inbound\"\n }\n ]\n },\n \"label\": \"Direction\"\n },\n {\n \"name\": \"common_entrance_id\",\n \"type\": \"int\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"Entrance ID\"\n },\n {\n \"name\": \"common_sled_ip\",\n \"type\": \"string\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n }\n },\n \"label\": \"Sled IP\"\n },\n {\n \"name\": \"common_client_location\",\n \"type\": \"string\",\n \"label\": \"Client Location\"\n },\n {\n \"name\": \"common_client_asn\",\n \"type\": \"string\",\n \"label\": \"Client ASN\"\n },\n {\n \"name\": \"common_server_location\",\n \"type\": \"string\",\n \"label\": \"Server Location\"\n },\n {\n \"name\": \"common_server_asn\",\n \"type\": \"string\",\n \"label\": \"Server ASN\"\n },\n {\n \"name\": \"common_sessions\",\n \"type\": \"long\",\n \"label\": \"Sessions\"\n },\n {\n \"name\": \"common_c2s_pkt_num\",\n \"type\": \"long\",\n \"label\": \"Packets Sent\"\n },\n {\n \"name\": \"common_s2c_pkt_num\",\n \"type\": \"long\",\n \"label\": \"Packets Received\"\n },\n {\n \"name\": \"common_c2s_byte_num\",\n \"type\": \"long\",\n \"label\": \"Bytes Sent\"\n },\n {\n \"name\": \"common_s2c_byte_num\",\n \"type\": \"long\",\n \"label\": \"Bytes Received\"\n },\n {\n \"name\": \"common_c2s_pkt_diff\",\n \"label\": \"Packets Sent(Diff)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_diff\",\n \"label\": \"Packets Received(Diff)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_diff\",\n \"label\": \"Bytes Sent(Diff)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_diff\",\n \"label\": \"Bytes Received(Diff)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_service\",\n \"type\": \"int\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"Service\"\n },\n {\n \"name\": \"common_schema_type\",\n \"type\": \"string\",\n \"doc\": {\n \"data\": [\n {\n \"code\": \"BASE\",\n \"value\": \"BASE\"\n },\n {\n \"code\": \"HTTP\",\n \"value\": \"HTTP\"\n },\n {\n \"code\": \"MAIL\",\n \"value\": \"MAIL\"\n },\n {\n \"code\": \"DNS\",\n \"value\": \"DNS\"\n },\n {\n \"code\": \"SSL\",\n \"value\": \"SSL\"\n },\n {\n \"code\": \"FTP\",\n \"value\": \"FTP\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"label\": \"Schema Type\"\n },\n {\n \"name\": \"common_user_tags\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"User Tags\"\n },\n {\n \"name\": \"common_sub_action\",\n \"type\": \"string\",\n \"doc\": {\n \"data\": [\n {\n \"code\": \"allow\",\n \"value\": \"Allow\"\n },\n {\n \"code\": \"deny\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"monitor\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"replace\",\n \"value\": \"Replace\"\n },\n {\n \"code\": \"redirect\",\n \"value\": \"Redirect\"\n },\n {\n \"code\": \"insert\",\n \"value\": \"Insert\"\n },\n {\n \"code\": \"hijack\",\n \"value\": \"Hijack\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"label\": \"Sub Action\"\n },\n {\n \"name\": \"common_user_region\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"User Region\"\n },\n {\n \"name\": \"common_device_id\",\n \"type\": \"string\",\n \"label\": \"Device ID\"\n },\n {\n \"name\": \"common_link_id\",\n \"type\": \"int\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"Link ID\"\n },\n {\n \"name\": \"common_isp\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"ISP\"\n },\n {\n \"name\": \"common_device_tag\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Device Tag\"\n },\n {\n \"name\": \"common_data_center\",\n \"type\": \"string\",\n \"label\": \"Data Center\"\n },\n {\n \"name\": \"common_encapsulation\",\n \"type\": \"int\",\n \"doc\": {\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"Ethernet\"\n },\n {\n \"code\": \"8\",\n \"value\": \"PPP\"\n },\n {\n \"code\": \"12\",\n \"value\": \"CiscoHDLC\"\n }\n ]\n },\n \"label\": \"Encapsulation\"\n },\n {\n \"name\": \"common_app_label\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"Application Label\"\n },\n {\n \"name\": \"common_tunnels\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Tunnels\"\n },\n {\n \"name\": \"common_protocol_label\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Protocol Label\"\n },\n {\n \"name\": \"common_app_id\",\n \"type\": \"string\",\n \"label\": \"Application ID\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_app_surrogate_id\",\n \"type\": \"string\",\n \"label\": \"Surrogate ID\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_l7_protocol\",\n \"type\": \"string\",\n \"label\": \"L7 Protocol\"\n },\n {\n \"name\": \"common_service_category\",\n \"label\": \"FQDN Category\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_start_time\",\n \"type\": \"long\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"visibility\": \"hidden\"\n },\n \"label\": \"Start Time\"\n },\n {\n \"name\": \"common_end_time\",\n \"type\": \"long\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"visibility\": \"hidden\"\n },\n \"label\": \"End Time\"\n },\n {\n \"name\": \"common_establish_latency_ms\",\n \"type\": \"long\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Establish Latency(ms)\"\n },\n {\n \"name\": \"common_con_duration_ms\",\n \"type\": \"long\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Duration(ms)\"\n },\n {\n \"name\": \"common_stream_dir\",\n \"type\": \"int\",\n \"doc\": {\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"c2s\"\n },\n {\n \"code\": \"2\",\n \"value\": \"s2c\"\n },\n {\n \"code\": \"3\",\n \"value\": \"double\"\n }\n ]\n },\n \"label\": \"Stream Direction\"\n },\n {\n \"name\": \"common_address_list\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"Address List\"\n },\n {\n \"name\": \"common_has_dup_traffic\",\n \"type\": \"int\",\n \"doc\": {\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"No\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Yes\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"label\": \"Duplication Traffic\"\n },\n {\n \"name\": \"common_stream_error\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Stream Error\"\n },\n {\n \"name\": \"common_stream_trace_id\",\n \"type\": \"long\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"label\": \"Session ID\"\n },\n {\n \"name\": \"common_link_info_c2s\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Link Info(c2s)\"\n },\n {\n \"name\": \"common_link_info_s2c\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Link Info(s2c)\"\n },\n {\n \"name\": \"common_c2s_ipfrag_num\",\n \"type\": \"long\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Fragmentation Packets(c2s)\"\n },\n {\n \"name\": \"common_s2c_ipfrag_num\",\n \"type\": \"long\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Fragmentation Packets(s2c)\"\n },\n {\n \"name\": \"common_c2s_tcp_lostlen\",\n \"type\": \"long\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Sequence Gap Loss(c2s)\"\n },\n {\n \"name\": \"common_s2c_tcp_lostlen\",\n \"type\": \"long\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Sequence Gap Loss(s2c)\"\n },\n {\n \"name\": \"common_c2s_tcp_unorder_num\",\n \"type\": \"long\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Unorder Packets(c2s)\"\n },\n {\n \"name\": \"common_s2c_tcp_unorder_num\",\n \"type\": \"long\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Unorder Packets(s2c)\"\n },\n {\n \"name\": \"common_c2s_pkt_retrans\",\n \"type\": \"long\",\n \"label\": \"Packet Retransmission(c2s)\"\n },\n {\n \"name\": \"common_s2c_pkt_retrans\",\n \"type\": \"long\",\n \"label\": \"Packet Retransmission(s2c)\"\n },\n {\n \"name\": \"common_c2s_byte_retrans\",\n \"type\": \"long\",\n \"label\": \"Byte Retransmission(c2s)\"\n },\n {\n \"name\": \"common_s2c_byte_retrans\",\n \"type\": \"long\",\n \"label\": \"Byte Retransmission(s2c)\"\n },\n {\n \"name\": \"common_tcp_client_isn\",\n \"label\": \"TCP Client ISN\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_tcp_server_isn\",\n \"label\": \"TCP Server ISN\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_first_ttl\",\n \"type\": \"int\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"First TTL\"\n },\n {\n \"name\": \"common_processing_time\",\n \"type\": \"long\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n }\n },\n \"label\": \"Processing Time\"\n },\n {\n \"name\": \"nic_name\",\n \"type\": \"string\",\n \"label\": \"Nic Name\"\n },\n {\n \"name\": \"origin_source_mac\",\n \"type\": \"string\",\n \"label\": \"Origin Source Mac\"\n },\n {\n \"name\": \"origin_dest_mac\",\n \"type\": \"string\",\n \"label\": \"Origin Dest Mac\"\n },\n {\n \"name\": \"packet_url\",\n \"type\": \"string\",\n \"label\": \"Packet URL\"\n },\n {\n \"name\": \"pcap_storage_task_id\",\n \"type\": \"int\",\n \"label\": \"Task ID\"\n }\n ]\n}', 'aef73490cd0c1dec63828fc6e5692c76', '2021-06-01 21:56:55', '2021-06-01 21:56:55', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (33, 'sys_storage_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"sys_storage_log\",\n \"namespace\": \"druid\",\n \"doc\": {\n \"partition_key\": \"__time\"\n },\n \"fields\": [\n {\n \"name\": \"__time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"log_type\",\n \"type\": \"string\"\n },\n {\n \"name\": \"data_center\",\n \"type\": \"string\"\n },\n {\n \"name\": \"max_size\",\n \"type\": \"long\"\n },\n {\n \"name\": \"used_size\",\n \"type\": \"long\"\n },\n {\n \"name\": \"aggregate_size\",\n \"type\": \"long\"\n },\n {\n \"name\": \"last_storage\",\n \"type\": \"long\"\n }\n ]\n}', '0d7bb3f54600ab4a125eb69b295debe8', '2021-06-01 21:56:56', '2021-06-01 21:56:56', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (34, 'tables_cluster.json', 'Galaxy', '{\n \"namespace\": \"system\",\n \"type\": \"record\",\n \"name\": \"tables_cluster\",\n \"fields\": [\n {\n \"name\": \"database\",\n \"type\": \"string\"\n }\n ]\n}', '33898898974182a090f4b837cd2d65de', '2021-06-01 21:56:57', '2021-06-01 21:56:57', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (35, 'top_client_ip_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"top_client_ip_log\",\n \"namespace\": \"druid\",\n \"doc\": {\n \"partition_key\": \"__time\"\n },\n \"fields\": [\n {\n \"name\": \"__time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"source\",\n \"type\": \"string\"\n },\n {\n \"name\": \"session_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"order_by\",\n \"type\": \"string\"\n },\n {\n \"name\": \"c2s_pkt_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"s2c_pkt_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"s2c_byte_num\",\n \"type\": \"long\"\n }\n ]\n}', '835429507f5224454a0d7646c7d690fa', '2021-06-01 21:56:58', '2021-06-01 21:56:58', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (36, 'top_external_host_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"top_external_host_log\",\n \"namespace\": \"druid\",\n \"doc\": {\n \"partition_key\": \"__time\"\n },\n \"fields\": [\n {\n \"name\": \"__time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_pkt_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"destination\",\n \"type\": \"string\"\n },\n {\n \"name\": \"order_by\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"s2c_pkt_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"session_num\",\n \"type\": \"long\"\n }\n ]\n}', 'f5ac6d1067d3b4281c66cdd5a3325e0e', '2021-06-01 21:56:59', '2021-06-01 21:56:59', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (37, 'top_internal_host_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"top_internal_host_log\",\n \"namespace\": \"druid\",\n \"doc\": {\n \"partition_key\": \"__time\"\n },\n \"fields\": [\n {\n \"name\": \"__time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_pkt_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"order_by\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"s2c_pkt_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"session_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"source\",\n \"type\": \"string\"\n }\n ]\n}', '4d7ee6d547776d08245b2f17aa1e0192', '2021-06-01 21:57:00', '2021-06-01 21:57:00', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (38, 'top_server_ip_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"top_server_ip_log\",\n \"namespace\": \"druid\",\n \"doc\": {\n \"partition_key\": \"__time\"\n },\n \"fields\": [\n {\n \"name\": \"__time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"destination\",\n \"type\": \"string\"\n },\n {\n \"name\": \"order_by\",\n \"type\": \"string\"\n },\n {\n \"name\": \"session_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_pkt_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"s2c_pkt_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"s2c_byte_num\",\n \"type\": \"long\"\n }\n ]\n}', 'c79b68646791521d2200652d307a8515', '2021-06-01 21:57:01', '2021-06-01 21:57:01', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (39, 'top_urls_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"top_urls_log\",\n \"namespace\": \"druid\",\n \"doc\": {\n \"partition_key\": \"__time\"\n },\n \"fields\": [\n {\n \"name\": \"__time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"session_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"url\",\n \"type\": \"string\"\n }\n ]\n}', '65163e1aca7459c918c8d8427a44d48e', '2021-06-01 21:57:01', '2021-06-01 21:57:01', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (40, 'top_user_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"top_user_log\",\n \"namespace\": \"druid\",\n \"doc\": {\n \"partition_key\": \"__time\"\n },\n \"fields\": [\n {\n \"name\": \"__time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_pkt_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"order_by\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"s2c_pkt_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"session_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"subscriber_id\",\n \"type\": \"string\"\n }\n ]\n}', 'ec1457cab68ac96c0442add5efa05acf', '2021-06-01 21:57:02', '2021-06-01 21:57:02', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (41, 'top_website_domain_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"top_website_domain_log\",\n \"namespace\": \"druid\",\n \"doc\": {\n \"partition_key\": \"__time\"\n },\n \"fields\": [\n {\n \"name\": \"__time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_pkt_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"domain\",\n \"type\": \"string\"\n },\n {\n \"name\": \"order_by\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"s2c_pkt_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"session_num\",\n \"type\": \"long\"\n }\n ]\n}', '178cc3826743b11e5e627c653bc924f1', '2021-06-01 21:57:03', '2021-06-01 21:57:03', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (42, 'traffic_metrics_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"traffic_metrics_log\",\n \"namespace\": \"druid\",\n \"doc\": {\n \"partition_key\": \"__time\"\n },\n \"fields\": [\n {\n \"name\": \"__time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"allow_conn_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"allow_in_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"allow_in_packets\",\n \"type\": \"long\"\n },\n {\n \"name\": \"allow_out_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"allow_out_packets\",\n \"type\": \"long\"\n },\n {\n \"name\": \"close_conn_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"default_conn_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"default_in_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"default_in_packets\",\n \"type\": \"long\"\n },\n {\n \"name\": \"default_out_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"default_out_packets\",\n \"type\": \"long\"\n },\n {\n \"name\": \"deny_conn_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"deny_in_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"deny_in_packets\",\n \"type\": \"long\"\n },\n {\n \"name\": \"deny_out_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"deny_out_packets\",\n \"type\": \"long\"\n },\n {\n \"name\": \"device_id\",\n \"type\": \"string\"\n },\n {\n \"name\": \"entrance_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"intercept_conn_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"intercept_in_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"intercept_in_packets\",\n \"type\": \"long\"\n },\n {\n \"name\": \"intercept_out_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"intercept_out_packets\",\n \"type\": \"long\"\n },\n {\n \"name\": \"established_conn_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"monitor_conn_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"monitor_in_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"monitor_in_packets\",\n \"type\": \"long\"\n },\n {\n \"name\": \"monitor_out_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"monitor_out_packets\",\n \"type\": \"long\"\n },\n {\n \"name\": \"new_conn_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"total_in_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"total_in_packets\",\n \"type\": \"long\"\n },\n {\n \"name\": \"total_out_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"total_out_packets\",\n \"type\": \"long\"\n },\n {\n \"name\": \"alert_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"hijk_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"ins_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"intcp_allow_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"intcp_deny_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"intcp_hijk_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"intcp_ins_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"intcp_mon_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"intcp_rdirt_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"intcp_repl_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"maybe_pinning_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"not_pinning_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"pinning_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"ad_cc_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"ad_flood_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"ad_reflection_bytes\",\n \"type\": \"long\"\n }\n ]\n}', 'a92128ff97187d9b18d8048ce003ba43', '2021-06-01 21:57:04', '2021-06-01 21:57:04', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (43, 'traffic_protocol_stat_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"traffic_protocol_stat_log\",\n \"namespace\": \"druid\",\n \"doc\": {\n \"partition_key\": \"__time\"\n },\n \"fields\": [\n {\n \"name\": \"__time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"protocol_id\",\n \"type\": \"string\"\n },\n {\n \"name\": \"isp\",\n \"type\": \"string\"\n },\n {\n \"name\": \"entrance_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"data_center\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sessions\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_pkt_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"s2c_pkt_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"s2c_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_ipfrag_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"s2c_ipfrag_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_tcp_lostlen\",\n \"type\": \"long\"\n },\n {\n \"name\": \"s2c_tcp_lostlen\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_tcp_unorder_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"s2c_tcp_unorder_num\",\n \"type\": \"long\"\n }\n ]\n}', '34c00f0852f3885a5aca1bd80a83783d', '2021-06-01 21:57:05', '2021-06-01 21:57:05', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (44, 'traffic_summary_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"traffic_summary_log\",\n \"namespace\": \"druid\",\n \"doc\": {\n \"partition_key\": \"__time\"\n },\n \"fields\": [\n {\n \"name\": \"__time\",\n \"type\": \"long\"\n },\n {\n \"name\": \"isp\",\n \"type\": \"string\"\n },\n {\n \"name\": \"entrance_id\",\n \"type\": \"long\"\n },\n {\n \"name\": \"data_center\",\n \"type\": \"string\"\n },\n {\n \"name\": \"schema_type\",\n \"type\": \"string\"\n },\n {\n \"name\": \"ip_object\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sessions\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_pkt_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"s2c_pkt_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"c2s_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"s2c_byte_num\",\n \"type\": \"long\"\n },\n {\n \"name\": \"one_sided_connections\",\n \"type\": \"long\"\n },\n {\n \"name\": \"uncategorized_bytes\",\n \"type\": \"long\"\n },\n {\n \"name\": \"fragmentation_packets\",\n \"type\": \"long\"\n },\n {\n \"name\": \"sequence_gap_loss\",\n \"type\": \"long\"\n },\n {\n \"name\": \"unorder_packets\",\n \"type\": \"long\"\n }\n ]\n}', '1252e6cb3800ac99ae64b98a830a8f13', '2021-06-01 21:57:06', '2021-06-01 21:57:06', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (45, 'transaction_record_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"transaction_record_log\",\n \"namespace\": \"tsg_galaxy_v3\",\n \"doc\": {\n \"primary_key\": \"common_stream_trace_id\",\n \"partition_key\": \"common_recv_time\",\n \"functions\": {\n \"aggregation\": [\n {\n \"name\": \"COUNT\",\n \"label\": \"COUNT\",\n \"function\": \"count(expr)\"\n },\n {\n \"name\": \"COUNT_DISTINCT\",\n \"label\": \"COUNT_DISTINCT\",\n \"function\": \"count(distinct expr)\"\n },\n {\n \"name\": \"AVG\",\n \"label\": \"AVG\",\n \"function\": \"avg(expr)\"\n },\n {\n \"name\": \"SUM\",\n \"label\": \"SUM\",\n \"function\": \"sum(expr)\"\n },\n {\n \"name\": \"MAX\",\n \"label\": \"MAX\",\n \"function\": \"max(expr)\"\n },\n {\n \"name\": \"MIN\",\n \"label\": \"MIN\",\n \"function\": \"min(expr)\"\n }\n ],\n \"operator\": [\n {\n \"name\": \"=\",\n \"label\": \"=\",\n \"function\": \"expr = value\"\n },\n {\n \"name\": \"!=\",\n \"label\": \"!=\",\n \"function\": \"expr != value\"\n },\n {\n \"name\": \">\",\n \"label\": \">\",\n \"function\": \"expr > value\"\n },\n {\n \"name\": \"<\",\n \"label\": \"<\",\n \"function\": \"expr < value\"\n },\n {\n \"name\": \">=\",\n \"label\": \">=\",\n \"function\": \"expr >= value\"\n },\n {\n \"name\": \"<=\",\n \"label\": \"<=\",\n \"function\": \"expr <= value\"\n },\n {\n \"name\": \"in\",\n \"label\": \"IN\",\n \"function\": \"expr in (values)\"\n },\n {\n \"name\": \"not in\",\n \"label\": \"NOT IN\",\n \"function\": \"expr not in (values)\"\n },\n {\n \"name\": \"like\",\n \"label\": \"LIKE\",\n \"function\": \"expr like value\"\n },\n {\n \"name\": \"not like\",\n \"label\": \"NOT LIKE\",\n \"function\": \"expr not like value\"\n },\n {\n \"name\": \"notEmpty\",\n \"label\": \"NOT EMPTY\",\n \"function\": \"notEmpty(expr)\"\n },\n {\n \"name\": \"empty\",\n \"label\": \"EMPTY\",\n \"function\": \"empty(expr)\"\n }\n ]\n },\n \"schema_query\": {\n \"dimensions\": [\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_client_location\",\n \"common_server_location\",\n \"common_subscriber_id\",\n \"common_client_port\",\n \"common_server_port\",\n \"common_schema_type\",\n \"common_l4_protocol\",\n \"common_l7_protocol\",\n \"common_data_center\",\n \"common_client_asn\",\n \"common_server_asn\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"http_host\",\n \"http_domain\",\n \"http_url\"\n ],\n \"metrics\": [\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_subscriber_id\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_sessions\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"http_host\",\n \"http_domain\",\n \"http_url\"\n ],\n \"filters\": [\n \"common_address_type\",\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_client_port\",\n \"common_server_port\",\n \"common_client_location\",\n \"common_server_location\",\n \"common_subscriber_id\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_l4_protocol\",\n \"common_l7_protocol\",\n \"common_stream_dir\",\n \"common_direction\",\n \"common_data_center\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_schema_type\",\n \"common_client_asn\",\n \"common_server_asn\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"http_host\",\n \"http_domain\",\n \"http_url\"\n ],\n \"references\": {\n \"aggregation\": [\n {\n \"type\": \"int\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"long\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"float\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"double\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"string\",\n \"functions\": \"COUNT,COUNT_DISTINCT\"\n },\n {\n \"type\": \"date\",\n \"functions\": \"COUNT,COUNT_DISTINCT,MAX,MIN\"\n },\n {\n \"type\": \"timestamp\",\n \"functions\": \"COUNT,COUNT_DISTINCT,MAX,MIN\"\n }\n ],\n \"operator\": [\n {\n \"type\": \"int\",\n \"functions\": \"=,!=,>,<,>=,<=,in,not in\"\n },\n {\n \"type\": \"long\",\n \"functions\": \"=,!=,>,<,>=,<=,in,not in\"\n },\n {\n \"type\": \"float\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"double\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"string\",\n \"functions\": \"=,!=,in,not in,like,not like,notEmpty,empty\"\n },\n {\n \"type\": \"date\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"timestamp\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n }\n ]\n },\n \"details\": {\n \"general\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_stream_trace_id\",\n \"common_direction\",\n \"common_stream_dir\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_processing_time\",\n \"common_entrance_id\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_data_center\",\n \"common_sled_ip\"\n ],\n \"action\": [\n \"common_action\",\n \"common_sub_action\",\n \"common_policy_id\",\n \"common_user_tags\",\n \"common_user_region\"\n ],\n \"source\": [\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_client_port\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\"\n ],\n \"destination\": [\n \"common_server_ip\",\n \"common_external_ip\",\n \"common_server_port\",\n \"common_server_location\",\n \"common_server_asn\"\n ],\n \"application\": [\n \"common_app_id\",\n \"common_app_label\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_protocol_label\",\n \"common_service_category\",\n \"common_service\",\n \"common_l4_protocol\"\n ],\n \"transmission\": [\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_first_ttl\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\"\n ],\n \"other\": [\n \"common_address_type\",\n \"common_schema_type\",\n \"common_device_tag\",\n \"common_encapsulation\",\n \"common_tunnels\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\"\n ]\n }\n },\n \"schema_type\": {\n \"BASE\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_service_category\",\n \"common_l7_protocol\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"HTTP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_service_category\",\n \"common_l7_protocol\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"http_url\",\n \"http_host\",\n \"http_domain\",\n \"http_request_line\",\n \"http_response_line\",\n \"http_request_header\",\n \"http_response_header\",\n \"http_request_content\",\n \"http_response_content\",\n \"http_request_body\",\n \"http_response_body\",\n \"http_request_body_key\",\n \"http_response_body_key\",\n \"http_proxy_flag\",\n \"http_sequence\",\n \"http_snapshot\",\n \"http_cookie\",\n \"http_referer\",\n \"http_user_agent\",\n \"http_content_length\",\n \"http_content_type\",\n \"http_set_cookie\",\n \"http_version\",\n \"http_response_lantency_ms\",\n \"http_session_duration_ms\",\n \"http_action_file_size\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"http_url\",\n \"common_server_port\"\n ]\n },\n \"MAIL\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\"\n ]\n },\n \"DNS\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"dns_message_id\",\n \"dns_qr\",\n \"dns_opcode\",\n \"dns_aa\",\n \"dns_tc\",\n \"dns_rd\",\n \"dns_ra\",\n \"dns_rcode\",\n \"dns_qdcount\",\n \"dns_ancount\",\n \"dns_nscount\",\n \"dns_arcount\",\n \"dns_qname\",\n \"dns_qtype\",\n \"dns_qclass\",\n \"dns_cname\",\n \"dns_sub\",\n \"dns_rr\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_client_ip\",\n \"dns_qr\",\n \"dns_qname\",\n \"dns_qtype\"\n ]\n },\n \"SSL\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"QUIC\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"FTP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"BGP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"SIP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"sip_call_id\",\n \"sip_originator_description\",\n \"sip_responder_description\",\n \"sip_user_agent\",\n \"sip_server\",\n \"sip_originator_sdp_connect_ip\",\n \"sip_originator_sdp_media_port\",\n \"sip_originator_sdp_media_type\",\n \"sip_originator_sdp_content\",\n \"sip_responder_sdp_connect_ip\",\n \"sip_responder_sdp_media_port\",\n \"sip_responder_sdp_media_type\",\n \"sip_responder_sdp_content\",\n \"sip_duration\",\n \"sip_bye\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"sip_originator_description\",\n \"sip_responder_description\",\n \"sip_call_id\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"RTP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"APP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_app_id\",\n \"common_app_label\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n }\n },\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_schema_type\"\n ],\n \"tunnel_type\": {\n \"GTP\": [\n {\n \"name\": \"gtp_sgw_ip\",\n \"label\": \"S-GW IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_pgw_ip\",\n \"label\": \"P-GW IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_sgw_port\",\n \"label\": \"S-GW Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"gtp_pgw_port\",\n \"label\": \"P-GW Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"gtp_uplink_teid\",\n \"label\": \"Uplink TEID\",\n \"type\": \"long\"\n },\n {\n \"name\": \"gtp_downlink_teid\",\n \"label\": \"Downlink TEID\",\n \"type\": \"long\"\n }\n ],\n \"MPLS\": [\n {\n \"name\": \"mpls_c2s_direction_label\",\n \"label\": \"Multiprotocol Label(c2s)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n },\n {\n \"name\": \"mpls_s2c_direction_label\",\n \"label\": \"Multiprotocol Label(s2c)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n }\n ],\n \"VLAN\": [\n {\n \"name\": \"vlan_c2s_direction_id\",\n \"label\": \"VLAN Direction(c2s)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n },\n {\n \"name\": \"vlan_s2c_direction_id\",\n \"label\": \"VLAN Direction(s2c)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n }\n ],\n \"ETHERNET\": [\n {\n \"name\": \"source_mac\",\n \"label\": \"Source MAC\",\n \"type\": \"string\"\n },\n {\n \"name\": \"destination_mac\",\n \"label\": \"Destination MAC\",\n \"type\": \"string\"\n }\n ],\n \"MULTIPATH_ETHERNET\": [\n {\n \"name\": \"c2s_source_mac\",\n \"label\": \"Source MAC(c2s)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"c2s_destination_mac\",\n \"label\": \"Destination MAC(c2s)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_source_mac\",\n \"label\": \"Source MAC(s2c)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_destination_mac\",\n \"label\": \"Destination MAC(s2c)\",\n \"type\": \"string\"\n }\n ]\n }\n },\n \"fields\": [\n {\n \"name\": \"common_recv_time\",\n \"type\": \"long\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"timestamp\"\n }\n },\n \"label\": \"Receive Time\"\n },\n {\n \"name\": \"common_log_id\",\n \"type\": \"long\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"snowflake_id\"\n }\n },\n \"label\": \"Log ID\"\n },\n {\n \"name\": \"common_policy_id\",\n \"type\": \"long\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Policy ID\"\n },\n {\n \"name\": \"common_subscriber_id\",\n \"type\": \"string\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"label\": \"Subscriber ID\"\n },\n {\n \"name\": \"common_imei\",\n \"type\": \"string\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"label\": \"IMEI\"\n },\n {\n \"name\": \"common_imsi\",\n \"type\": \"string\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"label\": \"IMSI\"\n },\n {\n \"name\": \"common_phone_number\",\n \"type\": \"string\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"label\": \"Phone Number\"\n },\n {\n \"name\": \"common_client_ip\",\n \"type\": \"string\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"geo_asn,radius_match\",\n \"appendTo\": \"common_client_asn,common_subscriber_id\"\n }\n },\n \"label\": \"Client IP\"\n },\n {\n \"name\": \"common_internal_ip\",\n \"type\": \"string\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"if\",\n \"param\": \"$.common_direction=69,$.common_client_ip,$.common_server_ip\"\n },\n \"allow_query\": \"true\"\n },\n \"label\": \"Internal IP\"\n },\n {\n \"name\": \"common_client_port\",\n \"type\": \"int\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"label\": \"Client Port\"\n },\n {\n \"name\": \"common_l4_protocol\",\n \"type\": \"string\",\n \"label\": \"L4 Protocol\"\n },\n {\n \"name\": \"common_address_type\",\n \"type\": \"int\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"4\",\n \"value\": \"ipv4\"\n },\n {\n \"code\": \"6\",\n \"value\": \"ipv6\"\n }\n ]\n },\n \"label\": \"Address Type\"\n },\n {\n \"name\": \"common_server_ip\",\n \"type\": \"string\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"geo_asn\",\n \"appendTo\": \"common_server_asn\"\n }\n },\n \"label\": \"Server IP\"\n },\n {\n \"name\": \"common_server_port\",\n \"type\": \"int\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"label\": \"Server Port\"\n },\n {\n \"name\": \"common_external_ip\",\n \"type\": \"string\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"if\",\n \"param\": \"$.common_direction=73,$.common_client_ip,$.common_server_ip\"\n },\n \"allow_query\": \"true\"\n },\n \"label\": \"External IP\"\n },\n {\n \"name\": \"common_action\",\n \"type\": \"int\",\n \"doc\": {\n \"visibility\": \"hidden\",\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"None\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Intercept\"\n },\n {\n \"code\": \"16\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"128\",\n \"value\": \"Allow\"\n }\n ]\n },\n \"label\": \"Action\"\n },\n {\n \"name\": \"common_direction\",\n \"type\": \"int\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"69\",\n \"value\": \"outbound\"\n },\n {\n \"code\": \"73\",\n \"value\": \"inbound\"\n }\n ]\n },\n \"label\": \"Direction\"\n },\n {\n \"name\": \"common_entrance_id\",\n \"type\": \"int\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"Entrance ID\"\n },\n {\n \"name\": \"common_sled_ip\",\n \"type\": \"string\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n }\n },\n \"label\": \"Sled IP\"\n },\n {\n \"name\": \"common_client_location\",\n \"type\": \"string\",\n \"label\": \"Client Location\"\n },\n {\n \"name\": \"common_client_asn\",\n \"type\": \"string\",\n \"label\": \"Client ASN\"\n },\n {\n \"name\": \"common_server_location\",\n \"type\": \"string\",\n \"label\": \"Server Location\"\n },\n {\n \"name\": \"common_server_asn\",\n \"type\": \"string\",\n \"label\": \"Server ASN\"\n },\n {\n \"name\": \"common_sessions\",\n \"type\": \"long\",\n \"label\": \"Sessions\"\n },\n {\n \"name\": \"common_c2s_pkt_num\",\n \"type\": \"long\",\n \"label\": \"Packets Sent\"\n },\n {\n \"name\": \"common_s2c_pkt_num\",\n \"type\": \"long\",\n \"label\": \"Packets Received\"\n },\n {\n \"name\": \"common_c2s_byte_num\",\n \"type\": \"long\",\n \"label\": \"Bytes Sent\"\n },\n {\n \"name\": \"common_s2c_byte_num\",\n \"type\": \"long\",\n \"label\": \"Bytes Received\"\n },\n {\n \"name\": \"common_c2s_pkt_diff\",\n \"type\": \"long\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Packets Sent(Diff)\"\n },\n {\n \"name\": \"common_s2c_pkt_diff\",\n \"type\": \"long\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Packets Received(Diff)\"\n },\n {\n \"name\": \"common_c2s_byte_diff\",\n \"type\": \"long\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Bytes Sent(Diff)\"\n },\n {\n \"name\": \"common_s2c_byte_diff\",\n \"type\": \"long\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Bytes Received(Diff)\"\n },\n {\n \"name\": \"common_service\",\n \"type\": \"int\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"Service\"\n },\n {\n \"name\": \"common_schema_type\",\n \"type\": \"string\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"BASE\",\n \"value\": \"BASE\"\n },\n {\n \"code\": \"DNS\",\n \"value\": \"DNS\"\n },\n {\n \"code\": \"HTTP\",\n \"value\": \"HTTP\"\n },\n {\n \"code\": \"SIP\",\n \"value\": \"SIP\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"label\": \"Schema Type\"\n },\n {\n \"name\": \"common_user_tags\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"User Tags\"\n },\n {\n \"name\": \"common_sub_action\",\n \"type\": \"string\",\n \"doc\": {\n \"data\": [\n {\n \"code\": \"allow\",\n \"value\": \"Allow\"\n },\n {\n \"code\": \"deny\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"monitor\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"replace\",\n \"value\": \"Replace\"\n },\n {\n \"code\": \"redirect\",\n \"value\": \"Redirect\"\n },\n {\n \"code\": \"insert\",\n \"value\": \"Insert\"\n },\n {\n \"code\": \"hijack\",\n \"value\": \"Hijack\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"label\": \"Sub Action\"\n },\n {\n \"name\": \"common_user_region\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"User Region\"\n },\n {\n \"name\": \"common_device_id\",\n \"type\": \"string\",\n \"label\": \"Device ID\"\n },\n {\n \"name\": \"common_link_id\",\n \"type\": \"int\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"Link ID\"\n },\n {\n \"name\": \"common_isp\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"ISP\"\n },\n {\n \"name\": \"common_device_tag\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Device Tag\"\n },\n {\n \"name\": \"common_data_center\",\n \"type\": \"string\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"City A\",\n \"value\": \"City A\"\n },\n {\n \"code\": \"City B\",\n \"value\": \"City B\"\n },\n {\n \"code\": \"City C\",\n \"value\": \"City C\"\n },\n {\n \"code\": \"City D\",\n \"value\": \"City D\"\n },\n {\n \"code\": \"City E\",\n \"value\": \"City E\"\n },\n {\n \"code\": \"City F\",\n \"value\": \"City F\"\n },\n {\n \"code\": \"City G\",\n \"value\": \"City G\"\n },\n {\n \"code\": \"City H\",\n \"value\": \"City H\"\n },\n {\n \"code\": \"City I\",\n \"value\": \"City I\"\n },\n {\n \"code\": \"City J\",\n \"value\": \"City J\"\n },\n {\n \"code\": \"City K\",\n \"value\": \"City K\"\n },\n {\n \"code\": \"City L\",\n \"value\": \"City L\"\n },\n {\n \"code\": \"City M\",\n \"value\": \"City M\"\n },\n {\n \"code\": \"City N\",\n \"value\": \"City N\"\n },\n {\n \"code\": \"City O\",\n \"value\": \"City O\"\n },\n {\n \"code\": \"City P\",\n \"value\": \"City P\"\n },\n {\n \"code\": \"City Q\",\n \"value\": \"City Q\"\n },\n {\n \"code\": \"City R\",\n \"value\": \"City R\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"label\": \"Data Center\"\n },\n {\n \"name\": \"common_encapsulation\",\n \"type\": \"int\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"Ethernet\"\n },\n {\n \"code\": \"8\",\n \"value\": \"PPP\"\n },\n {\n \"code\": \"12\",\n \"value\": \"CiscoHDLC\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"label\": \"Encapsulation\"\n },\n {\n \"name\": \"common_app_label\",\n \"type\": \"string\",\n \"label\": \"Application Label\"\n },\n {\n \"name\": \"common_tunnels\",\n \"type\": \"string\",\n \"label\": \"Tunnels\"\n },\n {\n \"name\": \"common_protocol_label\",\n \"type\": \"string\",\n \"label\": \"Protocol Label\"\n },\n {\n \"name\": \"common_app_id\",\n \"type\": \"string\",\n \"label\": \"Application ID\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_app_surrogate_id\",\n \"type\": \"string\",\n \"label\": \"Surrogate ID\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_l7_protocol\",\n \"type\": \"string\",\n \"label\": \"L7 Protocol\"\n },\n {\n \"name\": \"common_service_category\",\n \"type\": \"string\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"dict_location\": {\n \"path\": \"/v1/category/dict\",\n \"key\": \"categoryId\",\n \"value\": \"categoryName\"\n }\n },\n \"label\": \"FQDN Category\"\n },\n {\n \"name\": \"common_start_time\",\n \"type\": \"long\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n }\n },\n \"label\": \"Start Time\"\n },\n {\n \"name\": \"common_end_time\",\n \"type\": \"long\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"get_value\",\n \"appendTo\": \"common_recv_time\"\n }\n },\n \"label\": \"End Time\"\n },\n {\n \"name\": \"common_establish_latency_ms\",\n \"type\": \"long\",\n \"label\": \"Establish Latency(ms)\"\n },\n {\n \"name\": \"common_con_duration_ms\",\n \"type\": \"long\",\n \"label\": \"Duration(ms)\"\n },\n {\n \"name\": \"common_stream_dir\",\n \"type\": \"int\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"c2s\"\n },\n {\n \"code\": \"2\",\n \"value\": \"s2c\"\n },\n {\n \"code\": \"3\",\n \"value\": \"double\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"label\": \"Stream Direction\"\n },\n {\n \"name\": \"common_address_list\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"Address List\"\n },\n {\n \"name\": \"common_has_dup_traffic\",\n \"type\": \"int\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"No\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Yes\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"label\": \"Duplication Traffic\"\n },\n {\n \"name\": \"common_stream_error\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Stream Error\"\n },\n {\n \"name\": \"common_stream_trace_id\",\n \"type\": \"long\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"label\": \"Session ID\"\n },\n {\n \"name\": \"common_link_info_c2s\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Link Info(c2s)\"\n },\n {\n \"name\": \"common_link_info_s2c\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"Link Info(s2c)\"\n },\n {\n \"name\": \"common_c2s_ipfrag_num\",\n \"type\": \"long\",\n \"label\": \"Fragmentation Packets(c2s)\"\n },\n {\n \"name\": \"common_s2c_ipfrag_num\",\n \"type\": \"long\",\n \"label\": \"Fragmentation Packets(s2c)\"\n },\n {\n \"name\": \"common_c2s_tcp_lostlen\",\n \"type\": \"long\",\n \"label\": \"Sequence Gap Loss(c2s)\"\n },\n {\n \"name\": \"common_s2c_tcp_lostlen\",\n \"type\": \"long\",\n \"label\": \"Sequence Gap Loss(s2c)\"\n },\n {\n \"name\": \"common_c2s_tcp_unorder_num\",\n \"type\": \"long\",\n \"label\": \"Unorder Packets(c2s)\"\n },\n {\n \"name\": \"common_s2c_tcp_unorder_num\",\n \"type\": \"long\",\n \"label\": \"Unorder Packets(s2c)\"\n },\n {\n \"name\": \"common_c2s_pkt_retrans\",\n \"type\": \"long\",\n \"label\": \"Packet Retransmission(c2s)\"\n },\n {\n \"name\": \"common_s2c_pkt_retrans\",\n \"type\": \"long\",\n \"label\": \"Packet Retransmission(s2c)\"\n },\n {\n \"name\": \"common_c2s_byte_retrans\",\n \"type\": \"long\",\n \"label\": \"Byte Retransmission(c2s)\"\n },\n {\n \"name\": \"common_s2c_byte_retrans\",\n \"type\": \"long\",\n \"label\": \"Byte Retransmission(s2c)\"\n },\n {\n \"name\": \"common_tcp_client_isn\",\n \"type\": \"long\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"label\": \"TCP Client ISN\"\n },\n {\n \"name\": \"common_tcp_server_isn\",\n \"type\": \"long\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"label\": \"TCP Server ISN\"\n },\n {\n \"name\": \"common_first_ttl\",\n \"type\": \"int\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"First TTL\"\n },\n {\n \"name\": \"common_processing_time\",\n \"type\": \"long\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"current_timestamp\"\n }\n },\n \"label\": \"Processing Time\"\n },\n {\n \"name\": \"http_url\",\n \"type\": \"string\",\n \"label\": \"HTTP.URL\"\n },\n {\n \"name\": \"http_host\",\n \"type\": \"string\",\n \"doc\": {\n \"format\": {\n \"functions\": \"sub_domain\",\n \"appendTo\": \"http_domain\"\n }\n },\n \"label\": \"HTTP.Host\"\n },\n {\n \"name\": \"http_domain\",\n \"type\": \"string\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"label\": \"HTTP.Domain\"\n },\n {\n \"name\": \"http_request_line\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"HTTP.Request Line\"\n },\n {\n \"name\": \"http_response_line\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"HTTP.Response Line\"\n },\n {\n \"name\": \"http_request_header\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"HTTP.Request Headers\"\n },\n {\n \"name\": \"http_response_header\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"HTTP.Response Headers\"\n },\n {\n \"name\": \"http_request_content\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"HTTP.Request Content\"\n },\n {\n \"name\": \"http_response_content\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"HTTP.Response Content\"\n },\n {\n \"name\": \"http_request_body\",\n \"type\": \"string\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"file\"\n }\n },\n \"label\": \"HTTP.Request Body\"\n },\n {\n \"name\": \"http_response_body\",\n \"type\": \"string\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"file\"\n }\n },\n \"label\": \"HTTP.Response Body\"\n },\n {\n \"name\": \"http_request_body_key\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"HTTP.Request Body Key\"\n },\n {\n \"name\": \"http_response_body_key\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"HTTP.Response Body Key\"\n },\n {\n \"name\": \"http_proxy_flag\",\n \"type\": \"int\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"HTTP.Proxy Flag\"\n },\n {\n \"name\": \"http_sequence\",\n \"type\": \"int\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"HTTP.Sequence\"\n },\n {\n \"name\": \"http_snapshot\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"label\": \"HTTP.Snapshot\"\n },\n {\n \"name\": \"http_cookie\",\n \"type\": \"string\",\n \"label\": \"HTTP.Cookie\"\n },\n {\n \"name\": \"http_referer\",\n \"type\": \"string\",\n \"label\": \"HTTP.Referer\"\n },\n {\n \"name\": \"http_user_agent\",\n \"type\": \"string\",\n \"label\": \"HTTP.User Agent\"\n },\n {\n \"name\": \"http_content_length\",\n \"type\": \"string\",\n \"label\": \"HTTP.Content Length\"\n },\n {\n \"name\": \"http_content_type\",\n \"type\": \"string\",\n \"label\": \"HTTP.Content Type\"\n },\n {\n \"name\": \"http_set_cookie\",\n \"type\": \"string\",\n \"label\": \"HTTP.Set Cookie\"\n },\n {\n \"name\": \"http_version\",\n \"type\": \"string\",\n \"label\": \"HTTP.Version\"\n },\n {\n \"name\": \"http_response_lantency_ms\",\n \"type\": \"long\",\n \"label\": \"HTTP.Response Latency(ms)\"\n },\n {\n \"name\": \"http_session_duration_ms\",\n \"type\": \"long\",\n \"label\": \"HTTP.Session Duration(ms)\"\n },\n {\n \"name\": \"http_action_file_size\",\n \"type\": \"int\",\n \"label\": \"HTTP.Action File Size\"\n },\n {\n \"name\": \"dns_message_id\",\n \"type\": \"int\",\n \"label\": \"DNS.Message ID\"\n },\n {\n \"name\": \"dns_qr\",\n \"type\": \"int\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"QUERY\"\n },\n {\n \"code\": \"1\",\n \"value\": \"REESPONSE\"\n }\n ]\n },\n \"label\": \"DNS.QR\"\n },\n {\n \"name\": \"dns_opcode\",\n \"type\": \"int\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"QUERY\"\n },\n {\n \"code\": \"1\",\n \"value\": \"IQUERY\"\n },\n {\n \"code\": \"2\",\n \"value\": \"STATUS\"\n },\n {\n \"code\": \"5\",\n \"value\": \"UPDATE\"\n }\n ]\n },\n \"label\": \"DNS.OPCODE\"\n },\n {\n \"name\": \"dns_aa\",\n \"type\": \"int\",\n \"label\": \"DNS.AA\"\n },\n {\n \"name\": \"dns_tc\",\n \"type\": \"int\",\n \"label\": \"DNS.TC\"\n },\n {\n \"name\": \"dns_rd\",\n \"type\": \"int\",\n \"label\": \"DNS.RD\"\n },\n {\n \"name\": \"dns_ra\",\n \"type\": \"int\",\n \"label\": \"DNS.RA\"\n },\n {\n \"name\": \"dns_rcode\",\n \"type\": \"int\",\n \"label\": \"DNS.RCODE\"\n },\n {\n \"name\": \"dns_qdcount\",\n \"type\": \"int\",\n \"label\": \"DNS.QDCOUNT\"\n },\n {\n \"name\": \"dns_ancount\",\n \"type\": \"int\",\n \"label\": \"DNS.ANCOUNT\"\n },\n {\n \"name\": \"dns_nscount\",\n \"type\": \"int\",\n \"label\": \"DNS.NSCOUNT\"\n },\n {\n \"name\": \"dns_arcount\",\n \"type\": \"int\",\n \"label\": \"DNS.ARCOUNT\"\n },\n {\n \"name\": \"dns_qname\",\n \"type\": \"string\",\n \"label\": \"DNS.QNAME\"\n },\n {\n \"name\": \"dns_qtype\",\n \"type\": \"int\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"A\"\n },\n {\n \"code\": \"2\",\n \"value\": \"NS\"\n },\n {\n \"code\": \"5\",\n \"value\": \"CNAME\"\n },\n {\n \"code\": \"6\",\n \"value\": \"SOA\"\n },\n {\n \"code\": \"11\",\n \"value\": \"WKS\"\n },\n {\n \"code\": \"12\",\n \"value\": \"PTR\"\n },\n {\n \"code\": \"13\",\n \"value\": \"HINFO\"\n },\n {\n \"code\": \"11\",\n \"value\": \"WKS\"\n },\n {\n \"code\": \"15\",\n \"value\": \"MX\"\n },\n {\n \"code\": \"28\",\n \"value\": \"AAAA\"\n }\n ]\n },\n \"label\": \"DNS.QTYPE\"\n },\n {\n \"name\": \"dns_qclass\",\n \"type\": \"int\",\n \"label\": \"DNS.QCLASS\"\n },\n {\n \"name\": \"dns_cname\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"DNS.CNAME\"\n },\n {\n \"name\": \"dns_sub\",\n \"type\": \"int\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"DNS\"\n },\n {\n \"code\": \"2\",\n \"value\": \"DNSSEC\"\n }\n ]\n },\n \"label\": \"DNS.SUB\"\n },\n {\n \"name\": \"dns_rr\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"label\": \"DNS.RR\"\n },\n {\n \"name\": \"sip_call_id\",\n \"type\": \"string\",\n \"label\": \"SIP.Call-ID\"\n },\n {\n \"name\": \"sip_originator_description\",\n \"type\": \"string\",\n \"label\": \"SIP.Originator\"\n },\n {\n \"name\": \"sip_responder_description\",\n \"type\": \"string\",\n \"label\": \"SIP.Responder\"\n },\n {\n \"name\": \"sip_user_agent\",\n \"type\": \"string\",\n \"label\": \"SIP.User-Agent\"\n },\n {\n \"name\": \"sip_server\",\n \"type\": \"string\",\n \"label\": \"SIP.Server\"\n },\n {\n \"name\": \"sip_originator_sdp_connect_ip\",\n \"type\": \"string\",\n \"label\": \"SIP.Originator IP\"\n },\n {\n \"name\": \"sip_originator_sdp_media_port\",\n \"type\": \"int\",\n \"label\": \"SIP.Originator Port\"\n },\n {\n \"name\": \"sip_originator_sdp_media_type\",\n \"type\": \"string\",\n \"label\": \"SIP.Originator Media Type\"\n },\n {\n \"name\": \"sip_originator_sdp_content\",\n \"type\": \"string\",\n \"label\": \"SIP.Originator Content\"\n },\n {\n \"name\": \"sip_responder_sdp_connect_ip\",\n \"type\": \"string\",\n \"label\": \"SIP.Responder IP\"\n },\n {\n \"name\": \"sip_responder_sdp_media_port\",\n \"type\": \"int\",\n \"label\": \"SIP.Responder Port\"\n },\n {\n \"name\": \"sip_responder_sdp_media_type\",\n \"type\": \"string\",\n \"label\": \"SIP.Responder Media Type\"\n },\n {\n \"name\": \"sip_responder_sdp_content\",\n \"type\": \"string\",\n \"label\": \"SIP.Responder Content\"\n },\n {\n \"name\": \"sip_duration\",\n \"type\": \"int\",\n \"label\": \"SIP.Duration\"\n },\n {\n \"name\": \"sip_bye\",\n \"type\": \"string\",\n \"label\": \"SIP.Bye\"\n }\n ]\n}', '48f9fcdfc133cd8e6a346ef505e38c8f', '2021-06-01 21:57:07', '2021-06-01 21:57:07', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (46, 'version.json', 'Galaxy', '{\n \"product\": \"185 Environment\",\n \"version\": \"21.05\",\n \"registered\": \"Geedge\",\n \"updated\": \"2020-08-01 00:00:00\",\n \"components\": {\n \"oss\": [\n {\n \"name\": \"zookeeper\",\n \"version\": \"3.4.9\",\n \"licenseType\": \"Apache License 2.0\",\n \"description\": \"分布式应用程序协调服务\"\n },\n {\n \"name\": \"kafka\",\n \"version\": \"2.11_1.0.0\",\n \"licenseType\": \"Apache License 2.0\",\n \"description\": \"消息队列\"\n },\n {\n \"name\": \"habse\",\n \"version\": \"2.2.3\",\n \"licenseType\": \"Apache License 2.0\",\n \"description\": \"用于文件系统和存储Radius数据\"\n },\n {\n \"name\": \"flume\",\n \"version\": \"1.9.0\",\n \"licenseType\": \"Apache License 2.0\",\n \"description\": \"日志补全传输\"\n },\n {\n \"name\": \"clickhouse\",\n \"version\": \"19.17.4.11\",\n \"licenseType\": \"Apache License 2.0\",\n \"description\": \"原始日志数据库\"\n },\n {\n \"name\": \"druid\",\n \"version\": \"0.16.0\",\n \"licenseType\": \"Apache License 2.0\",\n \"description\": \"分析实时数据并提供低延迟查询的OLAP应用程序\"\n },\n {\n \"name\": \"gohangout\",\n \"version\": \"1.14.4.20200616\",\n \"description\": \"动态获取原始日志表schema入库程序\"\n }\n ],\n \"apps\": [\n {\n \"name\": \"galaxy-qgw-service\",\n \"version\": \"v3.2.200730\",\n \"description\": \"数据平台对外统一查询网关\"\n },\n {\n \"name\": \"galaxy-report-service\",\n \"version\": \"20.07.08\",\n \"description\": \"自定义报表查询服务\"\n },\n {\n \"name\": \"galaxy-hos-service\",\n \"version\": \"20.07.17\",\n \"description\": \"对象存储服务\"\n },\n {\n \"name\": \"xxl-job-admin\",\n \"version\": \"v1.0.2.20200506\",\n \"description\": \"分布式任务调度平台\"\n },\n {\n \"name\": \"xxl-job\",\n \"version\": \"v1.0.2.20200628\",\n \"description\": \"分布式任务调度平台-执行器\"\n }\n ],\n \"tasks\": [\n {\n \"name\": \"flume\",\n \"version\": \"flume-config-20.08\",\n \"description\": \"原始日志补全、subscriber更新、Radius上下线功能\"\n },\n {\n \"name\": \"druid\",\n \"version\": \"druid-config-20.08\",\n \"description\": \"所有分析日志任务\"\n },\n {\n \"name\": \"gohangout\",\n \"version\": \"gohangout-config-20.08\",\n \"description\": \"原始日志入库、上下线日志入库\"\n }\n ]\n }\n}', 'a7b4d71f3cffef8c0352e4a079c5a3b7', '2021-06-01 21:57:08', '2021-06-01 21:57:08', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); +INSERT INTO `config_info` VALUES (47, 'voip_record_log.json', 'Galaxy', '{\n \"type\": \"record\",\n \"name\": \"voip_record_log\",\n \"namespace\": \"tsg_galaxy_v3\",\n \"doc\": {\n \"primary_key\": \"common_log_id\",\n \"partition_key\": \"common_recv_time\",\n \"functions\": {\n \"aggregation\": [\n {\n \"name\": \"COUNT\",\n \"label\": \"COUNT\",\n \"function\": \"count(expr)\"\n },\n {\n \"name\": \"COUNT_DISTINCT\",\n \"label\": \"COUNT_DISTINCT\",\n \"function\": \"count(distinct expr)\"\n },\n {\n \"name\": \"AVG\",\n \"label\": \"AVG\",\n \"function\": \"avg(expr)\"\n },\n {\n \"name\": \"SUM\",\n \"label\": \"SUM\",\n \"function\": \"sum(expr)\"\n },\n {\n \"name\": \"MAX\",\n \"label\": \"MAX\",\n \"function\": \"max(expr)\"\n },\n {\n \"name\": \"MIN\",\n \"label\": \"MIN\",\n \"function\": \"min(expr)\"\n }\n ],\n \"operator\": [\n {\n \"name\": \"=\",\n \"label\": \"=\",\n \"function\": \"expr = value\"\n },\n {\n \"name\": \"!=\",\n \"label\": \"!=\",\n \"function\": \"expr != value\"\n },\n {\n \"name\": \">\",\n \"label\": \">\",\n \"function\": \"expr > value\"\n },\n {\n \"name\": \"<\",\n \"label\": \"<\",\n \"function\": \"expr < value\"\n },\n {\n \"name\": \">=\",\n \"label\": \">=\",\n \"function\": \"expr >= value\"\n },\n {\n \"name\": \"<=\",\n \"label\": \"<=\",\n \"function\": \"expr <= value\"\n },\n {\n \"name\": \"in\",\n \"label\": \"IN\",\n \"function\": \"expr in (values)\"\n },\n {\n \"name\": \"not in\",\n \"label\": \"NOT IN\",\n \"function\": \"expr not in (values)\"\n },\n {\n \"name\": \"like\",\n \"label\": \"LIKE\",\n \"function\": \"expr like value\"\n },\n {\n \"name\": \"not like\",\n \"label\": \"NOT LIKE\",\n \"function\": \"expr not like value\"\n },\n {\n \"name\": \"notEmpty\",\n \"label\": \"NOT EMPTY\",\n \"function\": \"notEmpty(expr)\"\n },\n {\n \"name\": \"empty\",\n \"label\": \"EMPTY\",\n \"function\": \"empty(expr)\"\n }\n ]\n },\n \"schema_query\": {\n \"dimensions\": [\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_client_location\",\n \"common_server_location\",\n \"common_subscriber_id\",\n \"common_client_port\",\n \"common_server_port\",\n \"common_schema_type\",\n \"common_l4_protocol\",\n \"common_l7_protocol\",\n \"common_data_center\",\n \"common_client_asn\",\n \"common_server_asn\",\n \"common_start_time\",\n \"common_end_time\",\n \"sip_call_id\",\n \"sip_originator_description\",\n \"sip_responder_description\",\n \"sip_user_agent\",\n \"sip_server\",\n \"sip_duration\",\n \"sip_bye\",\n \"rtp_payload_type_c2s\",\n \"rtp_payload_type_s2c\",\n \"rtp_originator_dir\"\n ],\n \"metrics\": [\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_subscriber_id\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_sessions\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"sip_call_id\",\n \"sip_originator_description\",\n \"sip_responder_description\",\n \"sip_user_agent\",\n \"sip_server\",\n \"sip_duration\"\n ],\n \"filters\": [\n \"common_address_type\",\n \"common_server_ip\",\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_external_ip\",\n \"common_client_port\",\n \"common_server_port\",\n \"common_client_location\",\n \"common_server_location\",\n \"common_subscriber_id\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_l4_protocol\",\n \"common_l7_protocol\",\n \"common_stream_dir\",\n \"common_direction\",\n \"common_data_center\",\n \"common_sled_ip\",\n \"common_device_id\",\n \"common_schema_type\",\n \"common_client_asn\",\n \"common_server_asn\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"sip_call_id\",\n \"sip_originator_description\",\n \"sip_responder_description\",\n \"sip_user_agent\",\n \"sip_server\",\n \"sip_duration\",\n \"sip_bye\",\n \"rtp_payload_type_c2s\",\n \"rtp_payload_type_s2c\",\n \"rtp_originator_dir\"\n ],\n \"references\": {\n \"aggregation\": [\n {\n \"type\": \"int\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"long\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"float\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"double\",\n \"functions\": \"COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN\"\n },\n {\n \"type\": \"string\",\n \"functions\": \"COUNT,COUNT_DISTINCT\"\n },\n {\n \"type\": \"date\",\n \"functions\": \"COUNT,COUNT_DISTINCT,MAX,MIN\"\n },\n {\n \"type\": \"timestamp\",\n \"functions\": \"COUNT,COUNT_DISTINCT,MAX,MIN\"\n }\n ],\n \"operator\": [\n {\n \"type\": \"int\",\n \"functions\": \"=,!=,>,<,>=,<=,in,not in\"\n },\n {\n \"type\": \"long\",\n \"functions\": \"=,!=,>,<,>=,<=,in,not in\"\n },\n {\n \"type\": \"float\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"double\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"string\",\n \"functions\": \"=,!=,in,not in,like,not like,notEmpty,empty\"\n },\n {\n \"type\": \"date\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n },\n {\n \"type\": \"timestamp\",\n \"functions\": \"=,!=,>,<,>=,<=\"\n }\n ]\n },\n \"details\": {\n \"general\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_stream_trace_id\",\n \"common_direction\",\n \"common_stream_dir\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_con_duration_ms\",\n \"common_establish_latency_ms\",\n \"common_processing_time\",\n \"common_entrance_id\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_data_center\",\n \"common_sled_ip\"\n ],\n \"action\": [\n \"common_action\",\n \"common_sub_action\",\n \"common_policy_id\",\n \"common_user_tags\",\n \"common_user_region\"\n ],\n \"source\": [\n \"common_client_ip\",\n \"common_internal_ip\",\n \"common_client_port\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\"\n ],\n \"destination\": [\n \"common_server_ip\",\n \"common_external_ip\",\n \"common_server_port\",\n \"common_server_location\",\n \"common_server_asn\"\n ],\n \"application\": [\n \"common_app_id\",\n \"common_app_label\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_protocol_label\",\n \"common_service_category\",\n \"common_service\",\n \"common_l4_protocol\"\n ],\n \"transmission\": [\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_first_ttl\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\"\n ],\n \"other\": [\n \"common_address_type\",\n \"common_schema_type\",\n \"common_device_tag\",\n \"common_encapsulation\",\n \"common_tunnels\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\"\n ]\n }\n },\n \"schema_type\": {\n \"SIP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"sip_call_id\",\n \"sip_originator_description\",\n \"sip_responder_description\",\n \"sip_user_agent\",\n \"sip_server\",\n \"sip_originator_sdp_connect_ip\",\n \"sip_originator_sdp_media_port\",\n \"sip_originator_sdp_media_type\",\n \"sip_originator_sdp_content\",\n \"sip_responder_sdp_connect_ip\",\n \"sip_responder_sdp_media_port\",\n \"sip_responder_sdp_media_type\",\n \"sip_responder_sdp_content\",\n \"sip_duration\",\n \"sip_bye\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"sip_originator_description\",\n \"sip_responder_description\",\n \"sip_call_id\",\n \"common_server_ip\",\n \"common_server_port\"\n ]\n },\n \"RTP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"rtp_payload_type_c2s\",\n \"rtp_payload_type_s2c\",\n \"rtp_pcap_path\",\n \"rtp_originator_dir\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"common_server_ip\",\n \"common_server_port\",\n \"rtp_pcap_path\",\n \"rtp_originator_dir\"\n ]\n },\n \"VoIP\": {\n \"columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_policy_id\",\n \"common_subscriber_id\",\n \"common_imei\",\n \"common_imsi\",\n \"common_phone_number\",\n \"common_client_ip\",\n \"common_client_port\",\n \"common_internal_ip\",\n \"common_l4_protocol\",\n \"common_address_type\",\n \"common_server_ip\",\n \"common_server_port\",\n \"common_external_ip\",\n \"common_action\",\n \"common_direction\",\n \"common_entrance_id\",\n \"common_sled_ip\",\n \"common_client_location\",\n \"common_client_asn\",\n \"common_server_location\",\n \"common_server_asn\",\n \"common_sessions\",\n \"common_c2s_pkt_num\",\n \"common_s2c_pkt_num\",\n \"common_c2s_byte_num\",\n \"common_s2c_byte_num\",\n \"common_c2s_pkt_diff\",\n \"common_s2c_pkt_diff\",\n \"common_c2s_byte_diff\",\n \"common_s2c_byte_diff\",\n \"common_service\",\n \"common_schema_type\",\n \"common_user_tags\",\n \"common_sub_action\",\n \"common_user_region\",\n \"common_device_id\",\n \"common_link_id\",\n \"common_isp\",\n \"common_device_tag\",\n \"common_data_center\",\n \"common_encapsulation\",\n \"common_app_label\",\n \"common_tunnels\",\n \"common_protocol_label\",\n \"common_app_id\",\n \"common_app_surrogate_id\",\n \"common_l7_protocol\",\n \"common_service_category\",\n \"common_start_time\",\n \"common_end_time\",\n \"common_establish_latency_ms\",\n \"common_con_duration_ms\",\n \"common_stream_dir\",\n \"common_address_list\",\n \"common_has_dup_traffic\",\n \"common_stream_error\",\n \"common_stream_trace_id\",\n \"common_link_info_c2s\",\n \"common_link_info_s2c\",\n \"common_c2s_ipfrag_num\",\n \"common_s2c_ipfrag_num\",\n \"common_c2s_tcp_lostlen\",\n \"common_s2c_tcp_lostlen\",\n \"common_c2s_tcp_unorder_num\",\n \"common_s2c_tcp_unorder_num\",\n \"common_c2s_pkt_retrans\",\n \"common_s2c_pkt_retrans\",\n \"common_c2s_byte_retrans\",\n \"common_s2c_byte_retrans\",\n \"common_tcp_client_isn\",\n \"common_tcp_server_isn\",\n \"common_first_ttl\",\n \"common_processing_time\",\n \"sip_call_id\",\n \"sip_originator_description\",\n \"sip_responder_description\",\n \"sip_user_agent\",\n \"sip_server\",\n \"sip_originator_sdp_connect_ip\",\n \"sip_originator_sdp_media_port\",\n \"sip_originator_sdp_media_type\",\n \"sip_originator_sdp_content\",\n \"sip_responder_sdp_connect_ip\",\n \"sip_responder_sdp_media_port\",\n \"sip_responder_sdp_media_type\",\n \"sip_responder_sdp_content\",\n \"sip_duration\",\n \"sip_bye\",\n \"rtp_payload_type_c2s\",\n \"rtp_payload_type_s2c\",\n \"rtp_pcap_path\",\n \"rtp_originator_dir\"\n ],\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"sip_originator_description\",\n \"sip_responder_description\",\n \"sip_call_id\",\n \"common_server_ip\",\n \"common_server_port\",\n \"rtp_pcap_path\",\n \"rtp_originator_dir\"\n ]\n }\n },\n \"default_columns\": [\n \"common_recv_time\",\n \"common_log_id\",\n \"common_subscriber_id\",\n \"common_client_ip\",\n \"sip_originator_description\",\n \"sip_responder_description\",\n \"sip_call_id\",\n \"common_server_ip\",\n \"common_server_port\",\n \"rtp_pcap_path\",\n \"rtp_originator_dir\"\n ],\n \"tunnel_type\": {\n \"GTP\": [\n {\n \"name\": \"gtp_sgw_ip\",\n \"label\": \"S-GW IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_pgw_ip\",\n \"label\": \"P-GW IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"gtp_sgw_port\",\n \"label\": \"S-GW Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"gtp_pgw_port\",\n \"label\": \"P-GW Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"gtp_uplink_teid\",\n \"label\": \"Uplink TEID\",\n \"type\": \"long\"\n },\n {\n \"name\": \"gtp_downlink_teid\",\n \"label\": \"Downlink TEID\",\n \"type\": \"long\"\n }\n ],\n \"MPLS\": [\n {\n \"name\": \"mpls_c2s_direction_label\",\n \"label\": \"Multiprotocol Label(c2s)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n },\n {\n \"name\": \"mpls_s2c_direction_label\",\n \"label\": \"Multiprotocol Label(s2c)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n }\n ],\n \"VLAN\": [\n {\n \"name\": \"vlan_c2s_direction_id\",\n \"label\": \"VLAN Direction(c2s)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n },\n {\n \"name\": \"vlan_s2c_direction_id\",\n \"label\": \"VLAN Direction(s2c)\",\n \"type\": {\n \"type\": \"array\",\n \"items\": \"int\"\n }\n }\n ],\n \"ETHERNET\": [\n {\n \"name\": \"source_mac\",\n \"label\": \"Source MAC\",\n \"type\": \"string\"\n },\n {\n \"name\": \"destination_mac\",\n \"label\": \"Destination MAC\",\n \"type\": \"string\"\n }\n ],\n \"MULTIPATH_ETHERNET\": [\n {\n \"name\": \"c2s_source_mac\",\n \"label\": \"Source MAC(c2s)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"c2s_destination_mac\",\n \"label\": \"Destination MAC(c2s)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_source_mac\",\n \"label\": \"Source MAC(s2c)\",\n \"type\": \"string\"\n },\n {\n \"name\": \"s2c_destination_mac\",\n \"label\": \"Destination MAC(s2c)\",\n \"type\": \"string\"\n }\n ]\n }\n },\n \"fields\": [\n {\n \"name\": \"common_recv_time\",\n \"label\": \"Receive Time\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_log_id\",\n \"label\": \"Log ID\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"format\": {\n \"functions\": \"snowflake_id\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_policy_id\",\n \"label\": \"Policy ID\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_subscriber_id\",\n \"label\": \"Subscriber ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_imei\",\n \"label\": \"IMEI\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_imsi\",\n \"label\": \"IMSI\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_phone_number\",\n \"label\": \"Phone Number\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_ip\",\n \"label\": \"Client IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"geo_asn,radius_match\",\n \"appendTo\": \"common_client_asn,common_subscriber_id\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_internal_ip\",\n \"label\": \"Internal IP\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"if\",\n \"param\": \"$.common_direction=69,$.common_client_ip,$.common_server_ip\"\n },\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_port\",\n \"label\": \"Client Port\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_l4_protocol\",\n \"label\": \"L4 Protocol\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_address_type\",\n \"label\": \"Address Type\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"4\",\n \"value\": \"ipv4\"\n },\n {\n \"code\": \"6\",\n \"value\": \"ipv6\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_server_ip\",\n \"label\": \"Server IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"geo_asn\",\n \"appendTo\": \"common_server_asn\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_port\",\n \"label\": \"Server Port\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_external_ip\",\n \"label\": \"External IP\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"ip\"\n },\n \"format\": {\n \"functions\": \"if\",\n \"param\": \"$.common_direction=73,$.common_client_ip,$.common_server_ip\"\n },\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_action\",\n \"label\": \"Action\",\n \"doc\": {\n \"visibility\": \"hidden\",\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"None\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"2\",\n \"value\": \"Intercept\"\n },\n {\n \"code\": \"16\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"128\",\n \"value\": \"Allow\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_direction\",\n \"label\": \"Direction\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"69\",\n \"value\": \"outbound\"\n },\n {\n \"code\": \"73\",\n \"value\": \"inbound\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_entrance_id\",\n \"label\": \"Entrance ID\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_sled_ip\",\n \"label\": \"Sled IP\",\n \"doc\": {\n \"allow_query\": \"true\",\n \"constraints\": {\n \"type\": \"ip\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_location\",\n \"label\": \"Client Location\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_client_asn\",\n \"label\": \"Client ASN\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_location\",\n \"label\": \"Server Location\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_server_asn\",\n \"label\": \"Server ASN\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sessions\",\n \"label\": \"Sessions\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_num\",\n \"label\": \"Packets Sent\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_num\",\n \"label\": \"Packets Received\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_num\",\n \"label\": \"Bytes Sent\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_num\",\n \"label\": \"Bytes Received\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_diff\",\n \"label\": \"Packets Sent(Diff)\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_diff\",\n \"label\": \"Packets Received(Diff)\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_diff\",\n \"label\": \"Bytes Sent(Diff)\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_diff\",\n \"label\": \"Bytes Received(Diff)\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_service\",\n \"label\": \"Service\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_schema_type\",\n \"label\": \"Schema Type\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"VoIP\",\n \"value\": \"VoIP\"\n },\n {\n \"code\": \"SIP\",\n \"value\": \"SIP\"\n },\n {\n \"code\": \"RTP\",\n \"value\": \"RTP\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_user_tags\",\n \"label\": \"User Tags\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_sub_action\",\n \"label\": \"Sub Action\",\n \"doc\": {\n \"data\": [\n {\n \"code\": \"allow\",\n \"value\": \"Allow\"\n },\n {\n \"code\": \"deny\",\n \"value\": \"Deny\"\n },\n {\n \"code\": \"monitor\",\n \"value\": \"Monitor\"\n },\n {\n \"code\": \"replace\",\n \"value\": \"Replace\"\n },\n {\n \"code\": \"redirect\",\n \"value\": \"Redirect\"\n },\n {\n \"code\": \"insert\",\n \"value\": \"Insert\"\n },\n {\n \"code\": \"hijack\",\n \"value\": \"Hijack\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_user_region\",\n \"label\": \"User Region\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_device_id\",\n \"label\": \"Device ID\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_link_id\",\n \"label\": \"Link ID\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_isp\",\n \"label\": \"ISP\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_device_tag\",\n \"label\": \"Device Tag\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_data_center\",\n \"label\": \"Data Center\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"City A\",\n \"value\": \"City A\"\n },\n {\n \"code\": \"City B\",\n \"value\": \"City B\"\n },\n {\n \"code\": \"City C\",\n \"value\": \"City C\"\n },\n {\n \"code\": \"City D\",\n \"value\": \"City D\"\n },\n {\n \"code\": \"City E\",\n \"value\": \"City E\"\n },\n {\n \"code\": \"City F\",\n \"value\": \"City F\"\n },\n {\n \"code\": \"City G\",\n \"value\": \"City G\"\n },\n {\n \"code\": \"City H\",\n \"value\": \"City H\"\n },\n {\n \"code\": \"City I\",\n \"value\": \"City I\"\n },\n {\n \"code\": \"City J\",\n \"value\": \"City J\"\n },\n {\n \"code\": \"City K\",\n \"value\": \"City K\"\n },\n {\n \"code\": \"City L\",\n \"value\": \"City L\"\n },\n {\n \"code\": \"City M\",\n \"value\": \"City M\"\n },\n {\n \"code\": \"City N\",\n \"value\": \"City N\"\n },\n {\n \"code\": \"City O\",\n \"value\": \"City O\"\n },\n {\n \"code\": \"City P\",\n \"value\": \"City P\"\n },\n {\n \"code\": \"City Q\",\n \"value\": \"City Q\"\n },\n {\n \"code\": \"City R\",\n \"value\": \"City R\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_encapsulation\",\n \"label\": \"Encapsulation\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"Ethernet\"\n },\n {\n \"code\": \"8\",\n \"value\": \"PPP\"\n },\n {\n \"code\": \"12\",\n \"value\": \"CiscoHDLC\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_app_label\",\n \"label\": \"Application Label\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_tunnels\",\n \"label\": \"Tunnels\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_protocol_label\",\n \"label\": \"Protocol Label\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_app_id\",\n \"label\": \"Application ID\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_app_surrogate_id\",\n \"label\": \"Surrogate ID\",\n \"type\": \"string\",\n \"doc\": {\n \"visibility\": \"hidden\"\n }\n },\n {\n \"name\": \"common_l7_protocol\",\n \"label\": \"L7 Protocol\",\n \"type\": \"string\"\n },\n {\n \"name\": \"common_service_category\",\n \"label\": \"FQDN Category\",\n \"doc\": {\n \"visibility\": \"disabled\",\n \"dict_location\": {\n \"path\": \"/v1/category/dict\",\n \"key\": \"categoryId\",\n \"value\": \"categoryName\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_start_time\",\n \"label\": \"Start Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_end_time\",\n \"label\": \"End Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"get_value\",\n \"appendTo\": \"common_recv_time\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_establish_latency_ms\",\n \"label\": \"Establish Latency(ms)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_con_duration_ms\",\n \"label\": \"Duration(ms)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_stream_dir\",\n \"label\": \"Stream Direction\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"1\",\n \"value\": \"c2s\"\n },\n {\n \"code\": \"2\",\n \"value\": \"s2c\"\n },\n {\n \"code\": \"3\",\n \"value\": \"double\"\n }\n ],\n \"allow_query\": \"true\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_address_list\",\n \"label\": \"Address List\",\n \"doc\": {\n \"visibility\": \"disabled\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_has_dup_traffic\",\n \"label\": \"Duplication Traffic\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"No\"\n },\n {\n \"code\": \"1\",\n \"value\": \"Yes\"\n }\n ],\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_stream_error\",\n \"label\": \"Stream Error\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_stream_trace_id\",\n \"label\": \"Session ID\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_link_info_c2s\",\n \"label\": \"Link Info(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_link_info_s2c\",\n \"label\": \"Link Info(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"common_c2s_ipfrag_num\",\n \"label\": \"Fragmentation Packets(c2s)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_ipfrag_num\",\n \"label\": \"Fragmentation Packets(s2c)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_tcp_lostlen\",\n \"label\": \"Sequence Gap Loss(c2s)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_tcp_lostlen\",\n \"label\": \"Sequence Gap Loss(s2c)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_tcp_unorder_num\",\n \"label\": \"Unorder Packets(c2s)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_tcp_unorder_num\",\n \"label\": \"Unorder Packets(s2c)\",\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_pkt_retrans\",\n \"label\": \"Packet Retransmission(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_pkt_retrans\",\n \"label\": \"Packet Retransmission(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_c2s_byte_retrans\",\n \"label\": \"Byte Retransmission(c2s)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_s2c_byte_retrans\",\n \"label\": \"Byte Retransmission(s2c)\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_tcp_client_isn\",\n \"label\": \"TCP Client ISN\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_tcp_server_isn\",\n \"label\": \"TCP Server ISN\",\n \"doc\": {\n \"allow_query\": \"true\"\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"common_first_ttl\",\n \"label\": \"First TTL\",\n \"doc\": {\n \"visibility\": \"hidden\"\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"common_processing_time\",\n \"label\": \"Processing Time\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"timestamp\"\n },\n \"format\": {\n \"functions\": \"current_timestamp\"\n }\n },\n \"type\": \"long\"\n },\n {\n \"name\": \"sip_call_id\",\n \"label\": \"SIP.Call-ID\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_originator_description\",\n \"label\": \"SIP.Originator\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_responder_description\",\n \"label\": \"SIP.Responder\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_user_agent\",\n \"label\": \"SIP.User-Agent\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_server\",\n \"label\": \"SIP.Server\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_originator_sdp_connect_ip\",\n \"label\": \"SIP.Originator IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_originator_sdp_media_port\",\n \"label\": \"SIP.Originator Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"sip_originator_sdp_media_type\",\n \"label\": \"SIP.Originator Media Type\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_originator_sdp_content\",\n \"label\": \"SIP.Originator Content\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_responder_sdp_connect_ip\",\n \"label\": \"SIP.Responder IP\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_responder_sdp_media_port\",\n \"label\": \"SIP.Responder Port\",\n \"type\": \"int\"\n },\n {\n \"name\": \"sip_responder_sdp_media_type\",\n \"label\": \"SIP.Responder Media Type\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_responder_sdp_content\",\n \"label\": \"SIP.Responder Content\",\n \"type\": \"string\"\n },\n {\n \"name\": \"sip_duration\",\n \"label\": \"SIP.Duration\",\n \"type\": \"int\"\n },\n {\n \"name\": \"sip_bye\",\n \"label\": \"SIP.Bye\",\n \"type\": \"string\"\n },\n {\n \"name\": \"rtp_payload_type_c2s\",\n \"label\": \"RTP.Payload Type(c2s)\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"PCMU\"\n },\n {\n \"code\": \"1\",\n \"value\": \"1016\"\n },\n {\n \"code\": \"2\",\n \"value\": \"G721\"\n },\n {\n \"code\": \"3\",\n \"value\": \"GSM\"\n },\n {\n \"code\": \"4\",\n \"value\": \"G723\"\n },\n {\n \"code\": \"5\",\n \"value\": \"DVI4_8000\"\n },\n {\n \"code\": \"6\",\n \"value\": \"DVI4_16000\"\n },\n {\n \"code\": \"7\",\n \"value\": \"LPC\"\n },\n {\n \"code\": \"8\",\n \"value\": \"PCMA\"\n },\n {\n \"code\": \"9\",\n \"value\": \"G722\"\n },\n {\n \"code\": \"10\",\n \"value\": \"L16_STEREO\"\n },\n {\n \"code\": \"11\",\n \"value\": \"L16_MONO\"\n },\n {\n \"code\": \"12\",\n \"value\": \"QCELP\"\n },\n {\n \"code\": \"13\",\n \"value\": \"CN\"\n },\n {\n \"code\": \"14\",\n \"value\": \"MPA\"\n },\n {\n \"code\": \"15\",\n \"value\": \"G728\"\n },\n {\n \"code\": \"16\",\n \"value\": \"DVI4_11025\"\n },\n {\n \"code\": \"17\",\n \"value\": \"DVI4_22050\"\n },\n {\n \"code\": \"18\",\n \"value\": \"G729\"\n },\n {\n \"code\": \"19\",\n \"value\": \"CN_OLD\"\n },\n {\n \"code\": \"25\",\n \"value\": \"CELB\"\n },\n {\n \"code\": \"26\",\n \"value\": \"JPEG\"\n },\n {\n \"code\": \"28\",\n \"value\": \"NV\"\n },\n {\n \"code\": \"31\",\n \"value\": \"H261\"\n },\n {\n \"code\": \"32\",\n \"value\": \"MPV\"\n },\n {\n \"code\": \"33\",\n \"value\": \"MP2T\"\n },\n {\n \"code\": \"34\",\n \"value\": \"H263\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"rtp_payload_type_s2c\",\n \"label\": \"RTP.Payload Type(s2c)\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"PCMU\"\n },\n {\n \"code\": \"1\",\n \"value\": \"1016\"\n },\n {\n \"code\": \"2\",\n \"value\": \"G721\"\n },\n {\n \"code\": \"3\",\n \"value\": \"GSM\"\n },\n {\n \"code\": \"4\",\n \"value\": \"G723\"\n },\n {\n \"code\": \"5\",\n \"value\": \"DVI4_8000\"\n },\n {\n \"code\": \"6\",\n \"value\": \"DVI4_16000\"\n },\n {\n \"code\": \"7\",\n \"value\": \"LPC\"\n },\n {\n \"code\": \"8\",\n \"value\": \"PCMA\"\n },\n {\n \"code\": \"9\",\n \"value\": \"G722\"\n },\n {\n \"code\": \"10\",\n \"value\": \"L16_STEREO\"\n },\n {\n \"code\": \"11\",\n \"value\": \"L16_MONO\"\n },\n {\n \"code\": \"12\",\n \"value\": \"QCELP\"\n },\n {\n \"code\": \"13\",\n \"value\": \"CN\"\n },\n {\n \"code\": \"14\",\n \"value\": \"MPA\"\n },\n {\n \"code\": \"15\",\n \"value\": \"G728\"\n },\n {\n \"code\": \"16\",\n \"value\": \"DVI4_11025\"\n },\n {\n \"code\": \"17\",\n \"value\": \"DVI4_22050\"\n },\n {\n \"code\": \"18\",\n \"value\": \"G729\"\n },\n {\n \"code\": \"19\",\n \"value\": \"CN_OLD\"\n },\n {\n \"code\": \"25\",\n \"value\": \"CELB\"\n },\n {\n \"code\": \"26\",\n \"value\": \"JPEG\"\n },\n {\n \"code\": \"28\",\n \"value\": \"NV\"\n },\n {\n \"code\": \"31\",\n \"value\": \"H261\"\n },\n {\n \"code\": \"32\",\n \"value\": \"MPV\"\n },\n {\n \"code\": \"33\",\n \"value\": \"MP2T\"\n },\n {\n \"code\": \"34\",\n \"value\": \"H263\"\n }\n ]\n },\n \"type\": \"int\"\n },\n {\n \"name\": \"rtp_pcap_path\",\n \"label\": \"RTP.PCAP\",\n \"doc\": {\n \"constraints\": {\n \"type\": \"files\"\n }\n },\n \"type\": \"string\"\n },\n {\n \"name\": \"rtp_originator_dir\",\n \"label\": \"RTP.Direction\",\n \"doc\": {\n \"constraints\": {\n \"operator_functions\": \"=,!=\"\n },\n \"data\": [\n {\n \"code\": \"0\",\n \"value\": \"unknown\"\n },\n {\n \"code\": \"1\",\n \"value\": \"c2s\"\n },\n {\n \"code\": \"2\",\n \"value\": \"s2c\"\n }\n ]\n },\n \"type\": \"int\"\n }\n ]\n}', 'e10f14061804cbe9595690813bc9d3cd', '2021-06-01 21:57:09', '2021-06-01 21:57:09', 'nacos', '192.168.40.203', 'galaxy-qgw-service', 'prod', NULL, NULL, NULL, 'json', NULL); + +-- ---------------------------- +-- Table structure for config_info_aggr +-- ---------------------------- +DROP TABLE IF EXISTS `config_info_aggr`; +CREATE TABLE `config_info_aggr` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id', + `data_id` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'data_id', + `group_id` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'group_id', + `datum_id` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'datum_id', + `content` longtext CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '内容', + `gmt_modified` datetime(0) NOT NULL COMMENT '修改时间', + `app_name` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL, + `tenant_id` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT '' COMMENT '租户字段', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE INDEX `uk_configinfoaggr_datagrouptenantdatum`(`data_id`, `group_id`, `tenant_id`, `datum_id`) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8 COLLATE = utf8_bin COMMENT = '增加租户字段' ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Table structure for config_info_beta +-- ---------------------------- +DROP TABLE IF EXISTS `config_info_beta`; +CREATE TABLE `config_info_beta` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id', + `data_id` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'data_id', + `group_id` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'group_id', + `app_name` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'app_name', + `content` longtext CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'content', + `beta_ips` varchar(1024) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'betaIps', + `md5` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'md5', + `gmt_create` datetime(0) NOT NULL DEFAULT current_timestamp() COMMENT '创建时间', + `gmt_modified` datetime(0) NOT NULL DEFAULT current_timestamp() COMMENT '修改时间', + `src_user` text CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'source user', + `src_ip` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'source ip', + `tenant_id` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT '' COMMENT '租户字段', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE INDEX `uk_configinfobeta_datagrouptenant`(`data_id`, `group_id`, `tenant_id`) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8 COLLATE = utf8_bin COMMENT = 'config_info_beta' ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Table structure for config_info_tag +-- ---------------------------- +DROP TABLE IF EXISTS `config_info_tag`; +CREATE TABLE `config_info_tag` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id', + `data_id` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'data_id', + `group_id` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'group_id', + `tenant_id` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT '' COMMENT 'tenant_id', + `tag_id` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'tag_id', + `app_name` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'app_name', + `content` longtext CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'content', + `md5` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'md5', + `gmt_create` datetime(0) NOT NULL DEFAULT current_timestamp() COMMENT '创建时间', + `gmt_modified` datetime(0) NOT NULL DEFAULT current_timestamp() COMMENT '修改时间', + `src_user` text CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'source user', + `src_ip` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'source ip', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE INDEX `uk_configinfotag_datagrouptenanttag`(`data_id`, `group_id`, `tenant_id`, `tag_id`) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8 COLLATE = utf8_bin COMMENT = 'config_info_tag' ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Table structure for config_tags_relation +-- ---------------------------- +DROP TABLE IF EXISTS `config_tags_relation`; +CREATE TABLE `config_tags_relation` ( + `id` bigint(20) NOT NULL COMMENT 'id', + `tag_name` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'tag_name', + `tag_type` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'tag_type', + `data_id` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'data_id', + `group_id` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'group_id', + `tenant_id` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT '' COMMENT 'tenant_id', + `nid` bigint(20) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`nid`) USING BTREE, + UNIQUE INDEX `uk_configtagrelation_configidtag`(`id`, `tag_name`, `tag_type`) USING BTREE, + INDEX `idx_tenant_id`(`tenant_id`) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8 COLLATE = utf8_bin COMMENT = 'config_tag_relation' ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Table structure for group_capacity +-- ---------------------------- +DROP TABLE IF EXISTS `group_capacity`; +CREATE TABLE `group_capacity` ( + `id` bigint(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `group_id` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Group ID，空字符表示整个集群', + `quota` int(10) UNSIGNED NOT NULL DEFAULT 0 COMMENT '配额，0表示使用默认值', + `usage` int(10) UNSIGNED NOT NULL DEFAULT 0 COMMENT '使用量', + `max_size` int(10) UNSIGNED NOT NULL DEFAULT 0 COMMENT '单个配置大小上限，单位为字节，0表示使用默认值', + `max_aggr_count` int(10) UNSIGNED NOT NULL DEFAULT 0 COMMENT '聚合子配置最大个数，，0表示使用默认值', + `max_aggr_size` int(10) UNSIGNED NOT NULL DEFAULT 0 COMMENT '单个聚合数据的子配置大小上限，单位为字节，0表示使用默认值', + `max_history_count` int(10) UNSIGNED NOT NULL DEFAULT 0 COMMENT '最大变更历史数量', + `gmt_create` datetime(0) NOT NULL DEFAULT current_timestamp() COMMENT '创建时间', + `gmt_modified` datetime(0) NOT NULL DEFAULT current_timestamp() COMMENT '修改时间', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE INDEX `uk_group_id`(`group_id`) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8 COLLATE = utf8_bin COMMENT = '集群、各Group容量信息表' ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Table structure for his_config_info +-- ---------------------------- +DROP TABLE IF EXISTS `his_config_info`; +CREATE TABLE `his_config_info` ( + `id` bigint(64) UNSIGNED NOT NULL, + `nid` bigint(20) UNSIGNED NOT NULL AUTO_INCREMENT, + `data_id` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `group_id` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `app_name` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'app_name', + `content` longtext CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `md5` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL, + `gmt_create` datetime(0) NOT NULL DEFAULT current_timestamp(), + `gmt_modified` datetime(0) NOT NULL DEFAULT current_timestamp(), + `src_user` text CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL, + `src_ip` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL, + `op_type` char(10) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL, + `tenant_id` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT '' COMMENT '租户字段', + PRIMARY KEY (`nid`) USING BTREE, + INDEX `idx_gmt_create`(`gmt_create`) USING BTREE, + INDEX `idx_gmt_modified`(`gmt_modified`) USING BTREE, + INDEX `idx_did`(`data_id`) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 75 CHARACTER SET = utf8 COLLATE = utf8_bin COMMENT = '多租户改造' ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Records of his_config_info +-- ---------------------------- + +-- ---------------------------- +-- Table structure for permissions +-- ---------------------------- +DROP TABLE IF EXISTS `permissions`; +CREATE TABLE `permissions` ( + `role` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + `resource` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + `action` varchar(8) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + UNIQUE INDEX `uk_role_permission`(`role`, `resource`, `action`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Table structure for roles +-- ---------------------------- +DROP TABLE IF EXISTS `roles`; +CREATE TABLE `roles` ( + `username` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + `role` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + UNIQUE INDEX `idx_user_role`(`username`, `role`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Records of roles +-- ---------------------------- +INSERT INTO `roles` VALUES ('nacos', 'ROLE_ADMIN'); + +-- ---------------------------- +-- Table structure for tenant_capacity +-- ---------------------------- +DROP TABLE IF EXISTS `tenant_capacity`; +CREATE TABLE `tenant_capacity` ( + `id` bigint(20) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `tenant_id` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'Tenant ID', + `quota` int(10) UNSIGNED NOT NULL DEFAULT 0 COMMENT '配额，0表示使用默认值', + `usage` int(10) UNSIGNED NOT NULL DEFAULT 0 COMMENT '使用量', + `max_size` int(10) UNSIGNED NOT NULL DEFAULT 0 COMMENT '单个配置大小上限，单位为字节，0表示使用默认值', + `max_aggr_count` int(10) UNSIGNED NOT NULL DEFAULT 0 COMMENT '聚合子配置最大个数', + `max_aggr_size` int(10) UNSIGNED NOT NULL DEFAULT 0 COMMENT '单个聚合数据的子配置大小上限，单位为字节，0表示使用默认值', + `max_history_count` int(10) UNSIGNED NOT NULL DEFAULT 0 COMMENT '最大变更历史数量', + `gmt_create` datetime(0) NOT NULL DEFAULT current_timestamp() COMMENT '创建时间', + `gmt_modified` datetime(0) NOT NULL DEFAULT current_timestamp() COMMENT '修改时间', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE INDEX `uk_tenant_id`(`tenant_id`) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8 COLLATE = utf8_bin COMMENT = '租户容量信息表' ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Table structure for tenant_info +-- ---------------------------- +DROP TABLE IF EXISTS `tenant_info`; +CREATE TABLE `tenant_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT 'id', + `kp` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'kp', + `tenant_id` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT '' COMMENT 'tenant_id', + `tenant_name` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT '' COMMENT 'tenant_name', + `tenant_desc` varchar(256) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'tenant_desc', + `create_source` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin NULL DEFAULT NULL COMMENT 'create_source', + `gmt_create` bigint(20) NOT NULL COMMENT '创建时间', + `gmt_modified` bigint(20) NOT NULL COMMENT '修改时间', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE INDEX `uk_tenant_info_kptenantid`(`kp`, `tenant_id`) USING BTREE, + INDEX `idx_tenant_id`(`tenant_id`) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 4 CHARACTER SET = utf8 COLLATE = utf8_bin COMMENT = 'tenant_info' ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Records of tenant_info +-- ---------------------------- +INSERT INTO `tenant_info` VALUES (1, '1', 'dev', 'dev', '开发环境', 'nacos', 1616642477210, 1616642477210); +INSERT INTO `tenant_info` VALUES (2, '1', 'test', 'test', '测试环境', 'nacos', 1616642477210, 1616642477210); +INSERT INTO `tenant_info` VALUES (3, '1', 'prod', 'prod', '生产环境', 'nacos', 1616642477210, 1616642477210); + +-- ---------------------------- +-- Table structure for users +-- ---------------------------- +DROP TABLE IF EXISTS `users`; +CREATE TABLE `users` ( + `username` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + `password` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + `enabled` tinyint(1) NOT NULL, + PRIMARY KEY (`username`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Records of users +-- ---------------------------- +INSERT INTO `users` VALUES ('nacos', '$2a$10$EuWPZHzz32dJN7jexM34MOeYirDdFAZm2kuWj7VEOJhhZkDrxfvUu', 1); + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/parcels/roles/components/nacos/templates/set-nacos-env.sh.j2 b/parcels/roles/components/nacos/templates/set-nacos-env.sh.j2 new file mode 100755 index 0000000..55e3068 --- /dev/null +++ b/parcels/roles/components/nacos/templates/set-nacos-env.sh.j2 @@ -0,0 +1,16 @@ +#!/bin/bash + +num=`cat /etc/profile | grep nacos | wc -l` + +if [ $num -eq "0" ];then + echo -e "\n#nacos" >> /etc/profile + echo -e "export NACOS_HOME={{ install_path }}/{{ nacos_version }}" >> /etc/profile + echo -e "export PATH=\$NACOS_HOME/bin:\$PATH" >> /etc/profile +fi +source /etc/profile +keeppath='/etc/init.d/keepnacosalive' +if [ -x $keeppath ];then + chkconfig --add keepnacosalive + chkconfig keepnacosalive on + service keepnacosalive start +fi diff --git a/parcels/roles/components/nacos/templates/startup.sh.j2 b/parcels/roles/components/nacos/templates/startup.sh.j2 new file mode 100755 index 0000000..740fa30 --- /dev/null +++ b/parcels/roles/components/nacos/templates/startup.sh.j2 @@ -0,0 +1,142 @@ +#!/bin/bash + +# Copyright 1999-2018 Alibaba Group Holding Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cygwin=false +darwin=false +os400=false +case "`uname`" in +CYGWIN*) cygwin=true;; +Darwin*) darwin=true;; +OS400*) os400=true;; +esac +error_exit () +{ + echo "ERROR: $1 !!" + exit 1 +} +[ ! -e "$JAVA_HOME/bin/java" ] && JAVA_HOME=$HOME/jdk/java +[ ! -e "$JAVA_HOME/bin/java" ] && JAVA_HOME=/usr/java +[ ! -e "$JAVA_HOME/bin/java" ] && JAVA_HOME=/opt/taobao/java +[ ! -e "$JAVA_HOME/bin/java" ] && unset JAVA_HOME + +if [ -z "$JAVA_HOME" ]; then + if $darwin; then + + if [ -x '/usr/libexec/java_home' ] ; then + export JAVA_HOME=`/usr/libexec/java_home` + + elif [ -d "/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home" ]; then + export JAVA_HOME="/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home" + fi + else + JAVA_PATH=`dirname $(readlink -f $(which javac))` + if [ "x$JAVA_PATH" != "x" ]; then + export JAVA_HOME=`dirname $JAVA_PATH 2>/dev/null` + fi + fi + if [ -z "$JAVA_HOME" ]; then + error_exit "Please set the JAVA_HOME variable in your environment, We need java(x64)! jdk8 or later is better!" + fi +fi + +export SERVER="nacos-server" +export MODE="cluster" +export FUNCTION_MODE="all" +export MEMBER_LIST="" +export EMBEDDED_STORAGE="" +while getopts ":m:f:s:c:p:" opt +do + case $opt in + m) + MODE=$OPTARG;; + f) + FUNCTION_MODE=$OPTARG;; + s) + SERVER=$OPTARG;; + c) + MEMBER_LIST=$OPTARG;; + p) + EMBEDDED_STORAGE=$OPTARG;; + ?) + echo "Unknown parameter" + exit 1;; + esac +done + +export JAVA_HOME +export JAVA="$JAVA_HOME/bin/java" +export BASE_DIR=`cd $(dirname $0)/..; pwd` +export CUSTOM_SEARCH_LOCATIONS=file:${BASE_DIR}/conf/ + +#=========================================================================================== +# JVM Configuration +#=========================================================================================== +if [[ "${MODE}" == "standalone" ]]; then + JAVA_OPT="${JAVA_OPT} -Xms512m -Xmx512m -Xmn256m" + JAVA_OPT="${JAVA_OPT} -Dnacos.standalone=true" +else + if [[ "${EMBEDDED_STORAGE}" == "embedded" ]]; then + JAVA_OPT="${JAVA_OPT} -DembeddedStorage=true" + fi + JAVA_OPT="${JAVA_OPT} -server {{ nacos_java_opt }} -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m" + JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${BASE_DIR}/logs/java_heapdump.hprof" + JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages" + JAVA_OPT="${JAVA_OPT} -Dnacos.server.ip={{ inventory_hostname }}" +fi + +if [[ "${FUNCTION_MODE}" == "config" ]]; then + JAVA_OPT="${JAVA_OPT} -Dnacos.functionMode=config" +elif [[ "${FUNCTION_MODE}" == "naming" ]]; then + JAVA_OPT="${JAVA_OPT} -Dnacos.functionMode=naming" +fi + +JAVA_OPT="${JAVA_OPT} -Dnacos.member.list=${MEMBER_LIST}" + +JAVA_MAJOR_VERSION=$($JAVA -version 2>&1 | sed -E -n 's/.* version "([0-9]*).*$/\1/p') +if [[ "$JAVA_MAJOR_VERSION" -ge "9" ]] ; then + JAVA_OPT="${JAVA_OPT} -Xlog:gc*:file=${BASE_DIR}/logs/nacos_gc.log:time,tags:filecount=10,filesize=102400" +else + JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${JAVA_HOME}/jre/lib/ext:${JAVA_HOME}/lib/ext" + JAVA_OPT="${JAVA_OPT} -Xloggc:${BASE_DIR}/logs/nacos_gc.log -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M" +fi + +JAVA_OPT="${JAVA_OPT} -Dloader.path=${BASE_DIR}/plugins/health,${BASE_DIR}/plugins/cmdb" +JAVA_OPT="${JAVA_OPT} -Dnacos.home=${BASE_DIR}" +JAVA_OPT="${JAVA_OPT} -jar ${BASE_DIR}/target/${SERVER}.jar" +JAVA_OPT="${JAVA_OPT} ${JAVA_OPT_EXT}" +JAVA_OPT="${JAVA_OPT} --spring.config.additional-location=${CUSTOM_SEARCH_LOCATIONS}" +JAVA_OPT="${JAVA_OPT} --logging.config=${BASE_DIR}/conf/nacos-logback.xml" +JAVA_OPT="${JAVA_OPT} --server.max-http-header-size=524288" + +if [ ! -d "${BASE_DIR}/logs" ]; then + mkdir ${BASE_DIR}/logs +fi + +echo "$JAVA ${JAVA_OPT}" + +if [[ "${MODE}" == "standalone" ]]; then + echo "nacos is starting with standalone" +else + echo "nacos is starting with cluster" +fi + +# check the start.out log output file +if [ ! -f "${BASE_DIR}/logs/start.out" ]; then + touch "${BASE_DIR}/logs/start.out" +fi +# start +echo "$JAVA ${JAVA_OPT}" > ${BASE_DIR}/logs/start.out 2>&1 & +nohup $JAVA ${JAVA_OPT} nacos.nacos >> ${BASE_DIR}/logs/start.out 2>&1 & +echo "nacos is starting，you can check the ${BASE_DIR}/logs/start.out" diff --git a/parcels/roles/components/nacos/templates/unload_nacos.sh.j2 b/parcels/roles/components/nacos/templates/unload_nacos.sh.j2 new file mode 100755 index 0000000..2740588 --- /dev/null +++ b/parcels/roles/components/nacos/templates/unload_nacos.sh.j2 @@ -0,0 +1,26 @@ +#!/bin/bash +source /etc/profile + +function killService(){ +keeppath='/etc/init.d/keepnacosalive' +if [ -x $keeppath ];then +service keepnacosalive stop +chkconfig keepnacosalive off +systemctl daemon-reload +rm -rf /etc/init.d/keepnacosalive +fi +} + +function killPid(){ +livenum=`jps -l | egrep -w "nacos-server" | grep -v grep |wc -l` +if [ $livenum -ne 0 ];then +keeppid=`jps -l |egrep -w "nacos-server" | awk '{print $1}'` +kill -9 $keeppid +fi +} + +killService +sleep 10 +killPid +rm -rf {{ install_path }}/{{ nacos_version }} +sleep 5 diff --git a/parcels/roles/components/nacos/vars/main.yml b/parcels/roles/components/nacos/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/components/nacos/vars/main.yml diff --git a/parcels/roles/components/spark/tasks/install_spark.yml b/parcels/roles/components/spark/tasks/install_spark.yml new file mode 100644 index 0000000..dc3c7cb --- /dev/null +++ b/parcels/roles/components/spark/tasks/install_spark.yml @@ -0,0 +1,192 @@ +- name: spark_nums to ansible variable + set_fact: spark_nums="{{groups.spark|length}}" + +- name: To terminate execution + fail: + msg: "spark 集群最少需要配置3个节点，请修改configurations/hosts内对应IP列表。" + when: spark_nums < '3' + +- name: check jdk 1.8_73 + shell: source /etc/profile && java -version 2>&1 | grep 1.8.0_73 | wc -l + ignore_errors: false + register: jdk_out + +- name: To terminate execution + fail: + msg: "检测到目标集群内存在JDK未安装节点，请检查" + when: jdk_out.stdout != '2' + run_once: true + delegate_to: 127.0.0.1 + +- name: create spark package path:{{ install_path }} + file: + state: directory + path: '{{ install_path }}' + +- block: + - name: Check if the Spark service already exists + shell: source /etc/profile && jps -l | grep "org.apache.spark.deploy" | grep -v grep | wc -l + register: check_out + - name: copy unload_spark.sh to {{ install_path }}/ + template: + src: 'unload_spark.sh.j2' + dest: '{{ install_path }}/unload_spark.sh' + force: true + mode: 0755 + when: check_out.stdout >= '1' + - name: unload spark + shell: cd {{ install_path }} && sh unload_spark.sh + when: check_out.stdout >= '1' + - name: Ansible delete {{ install_path }}/unload_spark.sh + file: + path: "{{ install_path }}/unload_spark.sh" + state: absent + - name: Check if the Spark service already exists + shell: source /etc/profile && jps -l | grep "org.apache.spark.deploy" | grep -v grep | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "卸载失败，组件可能非本安装部署，请手动卸载后继续安装" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "yes" + +- block: + - name: Check if the Spark service already exists + shell: source /etc/profile && jps -l | grep "org.apache.spark.deploy" | grep -v grep | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "检测到目标集群内已有Spark请检查IP是否正确或停止/卸载已启动的组件" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "no" + +- name: create scala package path:/usr/lib/scala + file: + state: directory + path: '/usr/lib/scala' + +#复制tar到目标服务器 +- name: copy {{ spark_version }}.tar.gz + copy: + src: '{{ package_path }}/{{ spark_version }}.tar.gz' + dest: '{{ install_path }}/' + force: true + +#复制tar到目标服务器 +- name: copy {{ scala_version }}.tar.gz + copy: + src: '{{ package_path }}/{{ scala_version }}.tar.gz' + dest: '/usr/lib/scala/' + force: true + +#解压tar +- name: unpack {{ spark_version }}.tar.gz + unarchive: + src: '{{ spark_version }}.tar.gz' + dest: '{{ install_path }}/' + +#解压tar +- name: unpack {{ scala_version }}.tar.gz + unarchive: + src: '{{ scala_version }}.tar.gz' + dest: '/usr/lib/scala/' + +- name: master_ip to ansible variable + set_fact: master_ip="{{groups.spark[0]}}" + +- name: set All IP to:{{ install_path }}/{{ spark_version }}/conf/slaves + shell: echo "{{ ansible_play_hosts }}" | grep -E -o "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+" > {{ install_path }}/{{ spark_version }}/conf/slaves + when: spark_nums <= '3' + +- name: set addition first IP to:{{ install_path }}/{{ spark_version }}/conf/slaves + shell: echo "{{ ansible_play_hosts }}" | grep -E -o "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+" | awk 'NR!=1 { print }' > {{ install_path }}/{{ spark_version }}/conf/slaves + when: spark_nums > '3' + +#拷贝spark 其他工作节点守护及开机自启脚本 +- name: copy spark config files + template: + src: 'spark-env.sh.j2' + dest: '{{ install_path }}/{{ spark_version }}/conf/spark-env.sh' + mode: 0755 + force: true + +#拷贝spark 其他工作节点守护及开机自启脚本 +- name: copy spark master dae/keep + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + force: true + with_items: + - { src: 'log4j-task.properties.j2', dest: '{{ install_path }}/{{ spark_version }}/conf/log4j-task.properties' } + - { src: 'spark-defaults.conf.j2', dest: '{{ install_path }}/{{ spark_version }}/conf/spark-defaults.conf' } + +#拷贝spark 其他工作节点守护及开机自启脚本 +- name: copy spark master dae/keep + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + mode: 0755 + with_items: + - { src: 'dae-sparkmaster.sh.j2', dest: '{{ install_path }}/{{ spark_version }}/sbin/dae-sparkmaster.sh' } + - { src: 'keepsparkmaster.j2', dest: '/etc/init.d/keepsparkmaster' } + when: spark_nums > '3' and inventory_hostname == master_ip + +#拷贝spark 其他工作节点守护及开机自启脚本 +- name: copy small spark master dae/keep + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + mode: 0755 + with_items: + - { src: 'dae-sparkall.sh.j2', dest: '{{ install_path }}/{{ spark_version }}/sbin/dae-sparkall.sh' } + - { src: 'keepsparkall.j2', dest: '/etc/init.d/keepsparkall' } + when: spark_nums <= '3' and inventory_hostname == master_ip + +#拷贝spark 其他工作节点守护及开机自启脚本 +- name: copy spark worker dae/keep + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + mode: 0755 + with_items: + - { src: 'dae-sparkworker.sh.j2', dest: '{{ install_path }}/{{ spark_version }}/sbin/dae-sparkworker.sh' } + - { src: 'keepsparkworker.j2', dest: '/etc/init.d/keepsparkworker' } + when: inventory_hostname != master_ip + +#拷贝 设置环境变量脚本 +- name: copy set_spark_env.sh path:{{ install_path }}/{{ spark_version }}/sbin/ + template: + src: 'set_spark_env.sh.j2' + dest: '{{ install_path }}/{{ spark_version }}/sbin/set_spark_env.sh' + mode: 0755 + +#配置环境变量 +- name: set spark env + shell: cd {{ install_path }}/{{ spark_version }}/sbin/ && ./set_spark_env.sh + +- name: start small spark master + shell: source /etc/profile && sh /etc/init.d/keepsparkall 'start' + when: spark_nums <= '3' and inventory_hostname == master_ip + +- name: start spark master + shell: source /etc/profile && sh /etc/init.d/keepsparkmaster 'start' + when: spark_nums > '3' and inventory_hostname == master_ip + +- name: start spark worker + shell: source /etc/profile && sh /etc/init.d/keepsparkworker 'start' + when: inventory_hostname != master_ip + +- name: Ansible delete {{ spark_version }}.tar.gz + file: + path: "{{ install_path }}/{{ spark_version }}.tar.gz" + state: absent + +- name: Ansible delete {{ scala_version }}.tar.gz + file: + path: "/usr/lib/scala/{{ scala_version }}.tar.gz" + state: absent + diff --git a/parcels/roles/components/spark/tasks/main.yml b/parcels/roles/components/spark/tasks/main.yml new file mode 100644 index 0000000..a0ceda3 --- /dev/null +++ b/parcels/roles/components/spark/tasks/main.yml @@ -0,0 +1 @@ +- include: install_spark.yml diff --git a/parcels/roles/components/spark/templates/dae-sparkall.sh.j2 b/parcels/roles/components/spark/templates/dae-sparkall.sh.j2 new file mode 100755 index 0000000..bd1d362 --- /dev/null +++ b/parcels/roles/components/spark/templates/dae-sparkall.sh.j2 @@ -0,0 +1,44 @@ +#!/bin/sh + +#修改BASE_DIR为安装的路径 +#启动命令端口可自行指定 +#JMX_PORT=9191 nohup $BASE_DIR/bin/kafka-server-start.sh $BASE_DIR/config/server.properties > /dev/null 2>&1 & + +BASE_DIR={{ install_path }} +VERSION={{ spark_version }} + +source /etc/profile + +while true ; do + ma=`jps -l | grep -w "org.apache.spark.deploy.master.Master" | grep -v grep |wc -l` + hi=`jps -l | grep -w "org.apache.spark.deploy.history.HistoryServer" | grep -v grep |wc -l` + wo=`jps -l | grep -w "org.apache.spark.deploy.worker.Worker" | grep -v grep |wc -l` + + if [ "$ma" -lt "1" ];then + $BASE_DIR/$VERSION/sbin/start-master.sh + OLD_NUM=`cat $BASE_DIR/$VERSION/protect/restartsum/Master` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASE_DIR/$VERSION/protect/restartsum/Master + echo "`date "+%Y-%m-%d %H:%M:%S"` - spark-master服务启动/异常重启 - 重启>次数 -> $RESTART_NUM" >> $BASE_DIR/$VERSION/protect/protectMaster.log + fi + + if [ "$hi" -lt "1" ];then + $BASE_DIR/$VERSION/sbin/start-history-server.sh + OLD_NUM=`cat $BASE_DIR/$VERSION/protect/restartsum/History` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASE_DIR/$VERSION/protect/restartsum/History + echo "`date "+%Y-%m-%d %H:%M:%S"` - spark-history服务启动/异常重启 - 重启>次数 -> $RESTART_NUM" >> $BASE_DIR/$VERSION/protect/protectHistory.log + fi + + if [ "$wo" -lt "1" ];then + $BASE_DIR/$VERSION/sbin/start-slave.sh spark://{{ master_ip }}:7077 + OLD_NUM=`cat $BASE_DIR/$VERSION/protect/restartsum/Worker` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASE_DIR/$VERSION/protect/restartsum/Worker + echo "`date "+%Y-%m-%d %H:%M:%S"` - spark-worker服务启动/异常重启 - 重启>次数 -> $RESTART_NUM" >> $BASE_DIR/$VERSION/protect/protectWorker.log + fi + + sleep 30 + +done + diff --git a/parcels/roles/components/spark/templates/dae-sparkmaster.sh.j2 b/parcels/roles/components/spark/templates/dae-sparkmaster.sh.j2 new file mode 100755 index 0000000..bd2db5d --- /dev/null +++ b/parcels/roles/components/spark/templates/dae-sparkmaster.sh.j2 @@ -0,0 +1,37 @@ +#!/bin/sh + +#修改BASE_DIR为安装的路径 +#启动命令端口可自行指定 +#JMX_PORT=9191 nohup $BASE_DIR/bin/kafka-server-start.sh $BASE_DIR/config/server.properties > /dev/null 2>&1 & + +PRO_NAME=org.apache.spark.deploy.master.Master +BASE_DIR={{ install_path }} +VERSION={{ spark_version }} +source /etc/profile + +while true ; do + ma=`jps -l | grep -w "org.apache.spark.deploy.master.Master" | grep -v grep |wc -l` + hi=`jps -l | grep -w "org.apache.spark.deploy.history.HistoryServer" | grep -v grep |wc -l` + + + if [ "$ma" -lt "1" ];then + $BASE_DIR/$VERSION/sbin/start-master.sh + OLD_NUM=`cat $BASE_DIR/$VERSION/protect/restartsum/Master` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASE_DIR/$VERSION/protect/restartsum/Master + echo "`date "+%Y-%m-%d %H:%M:%S"` - spark-master服务启动/异常重启 - 重启>次数 -> $RESTART_NUM" >> $BASE_DIR/$VERSION/protect/protectMaster.log + fi + + + if [ "$hi" -lt "1" ];then + $BASE_DIR/$VERSION/sbin/start-history-server.sh + OLD_NUM=`cat $BASE_DIR/$VERSION/protect/restartsum/History` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASE_DIR/$VERSION/protect/restartsum/History + echo "`date "+%Y-%m-%d %H:%M:%S"` - spark-history服务启动/异常重启 - 重启>次数 -> $RESTART_NUM" >> $BASE_DIR/$VERSION/protect/protectHistory.log + fi + + sleep 30 + +done + diff --git a/parcels/roles/components/spark/templates/dae-sparkworker.sh.j2 b/parcels/roles/components/spark/templates/dae-sparkworker.sh.j2 new file mode 100755 index 0000000..2de45ae --- /dev/null +++ b/parcels/roles/components/spark/templates/dae-sparkworker.sh.j2 @@ -0,0 +1,27 @@ +#!/bin/sh + +#修改BASE_DIR为安装的路径 +#启动命令端口可自行指定 +#JMX_PORT=9191 nohup $BASE_DIR/bin/kafka-server-start.sh $BASE_DIR/config/server.properties > /dev/null 2>&1 & + + +BASE_DIR={{ install_path }} +VERSION={{ spark_version }} + +source /etc/profile + +while true ; do + wo=`jps -l | grep -w "org.apache.spark.deploy.worker.Worker" | grep -v grep |wc -l` + + if [ "$wo" -lt "1" ];then + $BASE_DIR/$VERSION/sbin/start-slave.sh spark://{{ master_ip }}:7077 + OLD_NUM=`cat $BASE_DIR/$VERSION/protect/restartsum/Worker` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASE_DIR/$VERSION/protect/restartsum/Worker + echo "`date "+%Y-%m-%d %H:%M:%S"` - spark-worker服务启动/异常重启 - 重启>次数 -> $RESTART_NUM" >> $BASE_DIR/$VERSION/protect/protectWorker.log + fi + + sleep 30 + +done + diff --git a/parcels/roles/components/spark/templates/keepsparkall.j2 b/parcels/roles/components/spark/templates/keepsparkall.j2 new file mode 100755 index 0000000..b32fdf5 --- /dev/null +++ b/parcels/roles/components/spark/templates/keepsparkall.j2 @@ -0,0 +1,34 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 40 60 +# description: keepsparkall +source /etc/profile + +PRO_NAME=keepsparkall + +case $1 in +start) +keepnum=`ps -ef | grep dae-sparkall.sh.j2 | grep -v grep | wc -l` + +if [ $keepnum -eq "0" ];then +nohup {{ install_path }}/{{ spark_version }}/sbin/dae-sparkall.sh > /dev/null 2>&1 & +fi +;; + +stop) +keeppid=`ps -ef | grep dae-sparkall.sh | grep -v grep | awk '{print $2}'` +echo "守护进程PID：$keeppid" +kill -9 $keeppid +sleep 1 +sh {{ install_path }}/{{ spark_version }}/sbin/stop-history-server.sh +sh {{ install_path }}/{{ spark_version }}/sbin/stop-master.sh +sh {{ install_path }}/{{ spark_version }}/sbin/stop-slave.sh +;; + +* ) +echo "use keepsparkall [start|stop]" +;; +esac + diff --git a/parcels/roles/components/spark/templates/keepsparkmaster.j2 b/parcels/roles/components/spark/templates/keepsparkmaster.j2 new file mode 100755 index 0000000..268a081 --- /dev/null +++ b/parcels/roles/components/spark/templates/keepsparkmaster.j2 @@ -0,0 +1,33 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 40 60 +# description: keepsparkmaster +source /etc/profile + +PRO_NAME=keepsparkmaster + +case $1 in +start) +keepnum=`ps -ef | grep dae-sparkmaster.sh | grep -v grep | wc -l` + +if [ $keepnum -eq "0" ];then +nohup {{ install_path }}/{{ spark_version }}/sbin/dae-sparkmaster.sh > /dev/null 2>&1 & +fi +;; + +stop) +keeppid=`ps -ef | grep dae-sparkmaster.sh | grep -v grep | awk '{print $2}'` +echo "守护进程PID：$keeppid" +kill -9 $keeppid +sleep 1 +sh {{ install_path }}/{{ spark_version }}/sbin/stop-history-server.sh +sh {{ install_path }}/{{ spark_version }}/sbin/stop-master.sh +;; + +* ) +echo "use keepsparkmaster [start|stop]" +;; +esac + diff --git a/parcels/roles/components/spark/templates/keepsparkworker.j2 b/parcels/roles/components/spark/templates/keepsparkworker.j2 new file mode 100755 index 0000000..6124686 --- /dev/null +++ b/parcels/roles/components/spark/templates/keepsparkworker.j2 @@ -0,0 +1,32 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 40 60 +# description: keepsparkworker +source /etc/profile + +PRO_NAME=keepsparkworker + +case $1 in +start) +keepnum=`ps -ef | grep dae-sparkworker.sh | grep -v grep | wc -l` + +if [ $keepnum -eq "0" ];then +nohup {{ install_path }}/{{ spark_version }}/sbin/dae-sparkworker.sh > /dev/null 2>&1 & +fi +;; + +stop) +keeppid=`ps -ef | grep dae-sparkworker.sh | grep -v grep | awk '{print $2}'` +echo "守护进程PID：$keeppid" +kill -9 $keeppid +sleep 1 +sh {{ install_path }}/{{ spark_version }}/sbin/stop-slave.sh +;; + +* ) +echo "use keepsparkworker [start|stop]" +;; +esac + diff --git a/parcels/roles/components/spark/templates/log4j-task.properties.j2 b/parcels/roles/components/spark/templates/log4j-task.properties.j2 new file mode 100644 index 0000000..f7c5206 --- /dev/null +++ b/parcels/roles/components/spark/templates/log4j-task.properties.j2 @@ -0,0 +1,22 @@ +######################### logger ############################## +log4j.logger.org.apache.http=OFF +log4j.logger.org.apache.http.wire=OFF + +#Log4j +log4j.rootLogger=info,console,file + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=warn +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=[%d{yyyy-MM-dd HH\:mm\:ss}] [%-5p] [Thread\:%t] %l %x - <%m>%n + + +log4j.appender.file=org.apache.log4j.DailyRollingFileAppender +log4j.appender.file.Threshold=warn +log4j.appender.file.encoding=UTF-8 +log4j.appender.file.Append=true + +log4j.appender.file.file={{ install_path }}/{{ spark_version }}/logs/ip-learning-application.log +log4j.appender.file.DatePattern='.'yyyy-MM-dd +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.layout.ConversionPattern=[%d{yyyy-MM-dd HH\:mm\:ss}] [%-5p] %X{ip} [Thread\:%t] %l %x - %m%n diff --git a/parcels/roles/components/spark/templates/set_spark_env.sh.j2 b/parcels/roles/components/spark/templates/set_spark_env.sh.j2 new file mode 100755 index 0000000..e035754 --- /dev/null +++ b/parcels/roles/components/spark/templates/set_spark_env.sh.j2 @@ -0,0 +1,30 @@ +#!/bin/bash + +num=`cat /etc/profile | grep spark | wc -l` + +if [ $num -eq "0" ];then + echo -e "\n#spark" >> /etc/profile + echo -e "export SPARK_HOME={{ install_path }}/{{ spark_version }}" >> /etc/profile + echo -e "export PATH=\$SPARK_HOME/sbin:\$PATH" >> /etc/profile + echo -e "export PATH=\$SPARK_HOME/bin:\$PATH" >> /etc/profile + source /etc/profile +fi + +keeppath='/etc/init.d/keepsparkall' +if [ -x $keeppath ];then + chkconfig --add keepsparkall + chkconfig keepsparkall on +fi + +keeppath='/etc/init.d/keepsparkmaster' +if [ -x $keeppath ];then + chkconfig --add keepsparkmaster + chkconfig keepsparkmaster on +fi + +keeppath='/etc/init.d/keepsparkworker' +if [ -x $keeppath ];then + chkconfig --add keepsparkworker + chkconfig keepsparkworker on +fi + diff --git a/parcels/roles/components/spark/templates/spark-defaults.conf.j2 b/parcels/roles/components/spark/templates/spark-defaults.conf.j2 new file mode 100644 index 0000000..488800e --- /dev/null +++ b/parcels/roles/components/spark/templates/spark-defaults.conf.j2 @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Default system properties included when running spark-submit. +# This is useful for setting default environmental settings. + +# Example: +# spark.master spark://master:7077 +# spark.eventLog.enabled true +# spark.eventLog.dir hdfs://namenode:8021/directory +# spark.serializer org.apache.spark.serializer.KryoSerializer +# spark.driver.memory 5g +# spark.executor.extraJavaOptions -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three" +# + +spark.eventLog.enabled true +spark.local.dir {{ install_path }}/{{ spark_version }}/tmp/spark-events +spark.history.fs.logDirectory file:{{ install_path }}/{{ spark_version }}/tmp/spark-events +spark.eventLog.dir file:{{ install_path }}/{{ spark_version }}/tmp/spark-events +spark.history.fs.cleaner.enabled true +spark.history.fs.cleaner.interval 1d +spark.history.fs.cleaner.maxAge 7d +spark.executor.logs.rolling.strategy time +spark.executor.logs.rolling.time.interval daily +spark.executor.logs.rolling.maxRetainedFiles 10 diff --git a/parcels/roles/components/spark/templates/spark-env.sh.j2 b/parcels/roles/components/spark/templates/spark-env.sh.j2 new file mode 100755 index 0000000..62b5404 --- /dev/null +++ b/parcels/roles/components/spark/templates/spark-env.sh.j2 @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This file is sourced when running various Spark programs. +# Copy it as spark-env.sh and edit that to configure Spark for your site. + +# Options read when launching programs locally with +# ./bin/run-example or ./bin/spark-submit +# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files +# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node +# - SPARK_PUBLIC_DNS, to set the public dns name of the driver program + +# Options read by executors and drivers running inside the cluster +# - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node +# - SPARK_PUBLIC_DNS, to set the public DNS name of the driver program +# - SPARK_LOCAL_DIRS, storage directories to use on this node for shuffle and RDD data +# - MESOS_NATIVE_JAVA_LIBRARY, to point to your libmesos.so if you use Mesos + +# Options read in YARN client mode +# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files +# - SPARK_EXECUTOR_CORES, Number of cores for the executors (Default: 1). +# - SPARK_EXECUTOR_MEMORY, Memory per Executor (e.g. 1000M, 2G) (Default: 1G) +# - SPARK_DRIVER_MEMORY, Memory for Driver (e.g. 1000M, 2G) (Default: 1G) + +# Options for the daemons used in the standalone deploy mode +# - SPARK_MASTER_HOST, to bind the master to a different IP address or hostname +# - SPARK_MASTER_PORT / SPARK_MASTER_WEBUI_PORT, to use non-default ports for the master +# - SPARK_MASTER_OPTS, to set config properties only for the master (e.g. "-Dx=y") +# - SPARK_WORKER_CORES, to set the number of cores to use on this machine +# - SPARK_WORKER_MEMORY, to set how much total memory workers have to give executors (e.g. 1000m, 2g) +# - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT, to use non-default ports for the worker +# - SPARK_WORKER_DIR, to set the working directory of worker processes +# - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. "-Dx=y") +# - SPARK_DAEMON_MEMORY, to allocate to the master, worker and history server themselves (default: 1g). +# - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y") +# - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y") +# - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y") +# - SPARK_DAEMON_CLASSPATH, to set the classpath for all daemons +# - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers + +# Generic options for the daemons used in the standalone deploy mode +# - SPARK_CONF_DIR Alternate conf dir. (Default: ${SPARK_HOME}/conf) +# - SPARK_LOG_DIR Where log files are stored. (Default: ${SPARK_HOME}/logs) +# - SPARK_PID_DIR Where the pid file is stored. (Default: /tmp) +# - SPARK_IDENT_STRING A string representing this instance of spark. (Default: $USER) +# - SPARK_NICENESS The scheduling priority for daemons. (Default: 0) +# - SPARK_NO_DAEMONIZE Run the proposed command in the foreground. It will not output a PID file. + +export SPARK_MASTER_IP={{ master_ip }} +export SPARK_MASTER_PORT=7077 +export SPARK_MASTER_WEBUI_PORT=8088 +export SPARK_WORKER_CORES={{ spark_worker_cores }} +export SPARK_WORKER_INSTANCES=1 +export SPARK_WORKER_MEMORY={{ spark_worker_mem }}m +export JAVA_HOME={{ java_home }} +export SCALA_HOME={{ scala_home }} +export SPARK_WORKER_PORT=7070 + + +export SPARK_WORKER_OPTS=" +-Dspark.worker.cleanup.enabled=true +-Dspark.worker.cleanup.interval=86400 +-Dspark.worker.cleanup.appDataTtl=604800 +" diff --git a/parcels/roles/components/spark/templates/unload_spark.sh.j2 b/parcels/roles/components/spark/templates/unload_spark.sh.j2 new file mode 100755 index 0000000..44e8644 --- /dev/null +++ b/parcels/roles/components/spark/templates/unload_spark.sh.j2 @@ -0,0 +1,54 @@ +#!/bin/bash +source /etc/profile + +function killService(){ +keeppath='/etc/init.d/keepsparkall' +if [ -x $keeppath ];then +service keepsparkall stop +chkconfig keepsparkall off +systemctl daemon-reload +rm -rf /etc/init.d/keepsparkall +fi + +keeppath='/etc/init.d/keepsparkmaster' +if [ -x $keeppath ];then +service keepsparkmaster stop +chkconfig keepsparkmaster off +systemctl daemon-reload +rm -rf /etc/init.d/keepsparkmaster +fi + +keeppath='/etc/init.d/keepsparkworker' +if [ -x $keeppath ];then +service keepsparkworker stop +chkconfig keepsparkworker off +systemctl daemon-reload +rm -rf /etc/init.d/keepsparkworker +fi +} + +function killPid(){ +livenum=`jps -l | egrep -w "org.apache.spark.deploy.master.Master" | grep -v grep |wc -l` +if [ $livenum -ne 0 ];then +keeppid=`jps -l |egrep -w "org.apache.spark.deploy.master.Master" | awk '{print $1}'` +kill -9 $keeppid +fi + +livenum=`jps -l | egrep -w "org.apache.spark.deploy.history.HistoryServer" | grep -v grep |wc -l` +if [ $livenum -ne 0 ];then +keeppid=`jps -l |egrep -w "org.apache.spark.deploy.history.HistoryServer" | awk '{print $1}'` +kill -9 $keeppid +fi + +livenum=`jps -l | egrep -w "org.apache.spark.deploy.worker.Worker" | grep -v grep |wc -l` +if [ $livenum -ne 0 ];then +keeppid=`jps -l |egrep -w "org.apache.spark.deploy.worker.Worker" | awk '{print $1}'` +kill -9 $keeppid +fi +} + +killService +sleep 5 +killPid +rm -rf {{ install_path }}/{{ spark_version }} +sleep 5 diff --git a/parcels/roles/components/spark/vars/main.yml b/parcels/roles/components/spark/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/components/spark/vars/main.yml diff --git a/parcels/roles/components/storm/tasks/install_storm.yml b/parcels/roles/components/storm/tasks/install_storm.yml new file mode 100644 index 0000000..df3e022 --- /dev/null +++ b/parcels/roles/components/storm/tasks/install_storm.yml @@ -0,0 +1,185 @@ +- name: node_nums to ansible variable + set_fact: node_nums="{{groups.storm|length}}" + +- name: To terminate execution + fail: + msg: "storm 集群最少需要配置3个节点，请修改configurations/hosts内对应IP列表。" + when: node_nums < '3' + +- name: check jdk 1.8_73 + shell: source /etc/profile && java -version 2>&1 | grep 1.8.0_73 | wc -l + ignore_errors: false + register: jdk_out + +- name: To terminate execution + fail: + msg: "检测到目标集群内存在JDK未安装节点，请检查" + when: jdk_out.stdout != '2' + run_once: true + delegate_to: 127.0.0.1 + +- name: create storm package to {{ install_path }} + file: + state: directory + path: '{{ install_path }}' + +- block: + - name: Check if the Storm service already exists + shell: source /etc/profile && jps -l | egrep "org.apache.storm.daemon.nimbus|org.apache.storm.daemon.supervisor" | grep -v grep | wc -l + register: check_out + - name: copy unload_storm.sh to {{ install_path }}/ + template: + src: 'unload_storm.sh.j2' + dest: '{{ install_path }}/unload_storm.sh' + force: true + mode: 0755 + when: check_out.stdout >= '1' + - name: unload storm + shell: cd {{ install_path }} && sh unload_storm.sh + when: check_out.stdout >= '1' + - name: Ansible delete {{ install_path }}/unload_storm.sh + file: + path: "{{ install_path }}/unload_storm.sh" + state: absent + - name: Check if the Storm service already exists + shell: source /etc/profile && jps -l | egrep "org.apache.storm.daemon.nimbus|org.apache.storm.daemon.supervisor" | grep -v grep | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "卸载失败，组件可能非本安装部署，请手动卸载后继续安装" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "yes" + +- block: + - name: Check if the Storm service already exists + shell: source /etc/profile && jps -l | egrep "org.apache.storm.daemon.nimbus|org.apache.storm.daemon.supervisor" | grep -v grep | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "检测到目标集群内已有Storm请检查IP是否正确或停止/卸载已启动的组件" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "no" + +#设置为ansible变量 +- name: master_ip to ansible variable + set_fact: master_ip="{{groups.storm[0]}}" + +- name: slave1_ip to ansible variable + set_fact: slave1_ip="{{groups.storm[1]}}" + when: node_nums > '3' + +- name: slave2_ip to ansible variable + set_fact: slave2_ip="{{groups.storm[2]}}" + when: node_nums > '3' + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181" + when: '(groups.zookeeper|length) == 3' + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181,{{groups.zookeeper[3]}}:2181,{{groups.zookeeper[4]}}:2181" + when: '(groups.zookeeper|length) >= 5' + +- name: create storm package path:{{ install_path }}/storm + file: + state: directory + path: '{{ install_path }}/storm' + +- name: copy apache-storm-1.0.2.tar.gz to {{ install_path }}/ + copy: + src: '{{ package_path }}/apache-storm-1.0.2.tar.gz' + dest: '{{ install_path }}/' + force: true + +- name: unpack apache-storm-1.0.2.tar.gz + unarchive: + src: 'apache-storm-1.0.2.tar.gz' + dest: '{{ install_path }}/' + +- name: copy dae-nimbus.sh to {{ install_path }}/{{ storm_version }}/bin/ + template: + src: 'dae-nimbus.sh.j2' + dest: '{{ install_path }}/{{ storm_version }}/bin/dae-nimbus.sh' + mode: 0755 + force: true + +- name: copy dae-supervisor.sh to {{ install_path }}/{{ storm_version }}/bin/ + template: + src: 'dae-supervisor.sh.j2' + dest: '{{ install_path }}/{{ storm_version }}/bin/dae-supervisor.sh' + mode: 0755 + force: true + +#获取 ip 列表 +- name: get storm numbus iplist + shell: echo "{{ ansible_play_hosts }}" | grep -E -o "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+" > {{ install_path }}/{{ storm_version }}/conf/ip_list + +- name: copy keepnimalive to master nimbus + template: + src: 'keepnimalive.j2' + dest: '/etc/init.d/keepnimalive' + mode: 0755 + force: true + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + when: node_nums <= '3' + +- name: copy keepnimalive to all nimbus + template: + src: 'keepnimalive.j2' + dest: '/etc/init.d/keepnimalive' + mode: 0755 + force: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ master_ip }}' } + - { ip: '{{ slave1_ip }}' } + - { ip: '{{ slave2_ip }}' } + when: node_nums > '3' + +- name: copy keepsupalive path:/etc/init + template: + src: 'keepsupalive.j2' + dest: '/etc/init.d/keepsupalive' + mode: 0755 + force: true + +- name: copy worker.xml path:{{ install_path }}/{{ storm_version }}/log4j2/ + template: + src: 'worker.xml.j2' + dest: '{{ install_path }}/{{ storm_version }}/log4j2/worker.xml' + + force: true + +- name: copy set_storm_env.sh path:{{ install_path }}/{{ storm_version }}/bin/ + template: + src: 'set_storm_env.sh.j2' + dest: '{{ install_path }}/{{ storm_version }}/bin/set_storm_env.sh' + mode: 0755 + force: true + +- name: copy set_storm_conf.sh path:{{ install_path }}/{{ storm_version }}/conf/ + template: + src: 'set_storm_conf.sh.j2' + dest: '{{ install_path }}/{{ storm_version }}/conf/set_storm_conf.sh' + mode: 0755 + force: true + +#配置系统配置 +- name: set storm conf + shell: cd {{ install_path }}/{{ storm_version }}/conf/ && ./set_storm_conf.sh + +#配置环境变量 +- name: set storm env and chkconfig keepnimalive/keepsupalive + shell: cd {{ install_path }}/{{ storm_version }}/bin/ && ./set_storm_env.sh + +- name: Ansible delete {{ storm_version }}.tar.gz + file: + path: "{{ install_path }}/{{ storm_version }}.tar.gz" + state: absent diff --git a/parcels/roles/components/storm/tasks/install_storm.yml_bak b/parcels/roles/components/storm/tasks/install_storm.yml_bak new file mode 100644 index 0000000..2105d34 --- /dev/null +++ b/parcels/roles/components/storm/tasks/install_storm.yml_bak @@ -0,0 +1,101 @@ +- name: Check if the zookeeper service already exists + shell: source /etc/profile && jps -l | egrep "org.apache.storm.daemon.nimbus|org.apache.storm.daemon.supervisor" | grep -v grep | wc -l + register: check_nim_out + when: check_storm == "yes" + +- fail: + msg: "检测目标服务器已启动storm进程，请检查" + when: check_nim_out.stdout >= '1' and check_storm == "yes" + +- name: create storm package path:{{ install_path }} + file: + state: directory + path: '{{ install_path }}' + +- name: create storm package path:{{ install_path }}/storm + file: + state: directory + path: '{{ install_path }}/storm' + +#复制tar到目标服务器 +- name: copy apache-storm-1.0.2.tar.gz path:{{ install_path }} + copy: + src: '{{ package_path }}/apache-storm-1.0.2.tar.gz' + dest: '{{ install_path }}/' + force: true + backup: yes + +#解压tar +- name: unpack apache-storm-1.0.2.tar.gz + unarchive: + src: 'apache-storm-1.0.2.tar.gz' + dest: '{{ install_path }}/' + +- name: copy dae-nimbus.sh path:{{ install_path }}/{{ storm_version }}/bin/ + template: + src: 'dae-nimbus.sh.j2' + dest: '{{ install_path }}/{{ storm_version }}/bin/dae-nimbus.sh' + mode: 0755 + backup: yes + +- name: copy dae-supervisor.sh path:{{ install_path }}/{{ storm_version }}/bin/ + template: + src: 'dae-supervisor.sh.j2' + dest: '{{ install_path }}/{{ storm_version }}/bin/dae-supervisor.sh' + mode: 0755 + backup: yes + +- name: copy keepnimalive path:/etc/init + template: + src: 'keepnimalive.j2' + dest: '/etc/init.d/keepnimalive' + mode: 0755 + backup: yes + when: status=="master" + +- name: copy keepsupalive path:/etc/init + template: + src: 'keepsupalive.j2' + dest: '/etc/init.d/keepsupalive' + mode: 0755 + backup: yes + +- name: copy worker.xml path:{{ install_path }}/{{ storm_version }}/log4j2/ + template: + src: 'worker.xml.j2' + dest: '{{ install_path }}/{{ storm_version }}/log4j2/worker.xml' + backup: yes + +- name: copy set_storm_env.sh path:{{ install_path }}/{{ storm_version }}/bin/ + template: + src: 'set_storm_env.sh.j2' + dest: '{{ install_path }}/{{ storm_version }}/bin/set_storm_env.sh' + mode: 0755 + backup: yes + +#- name: copy worker.xml path:{{ install_path }}/{{ storm_version }}/conf +# template: +# src: 'storm.yaml.j2' +# dest: '{{ install_path }}/{{ storm_version }}/conf/storm.yaml' +# backup: yes + +- name: copy set_storm_conf.sh path:{{ install_path }}/{{ storm_version }}/conf/ + template: + src: 'set_storm_conf.sh.j2' + dest: '{{ install_path }}/{{ storm_version }}/conf/set_storm_conf.sh' + mode: 0755 + backup: yes + +#配置环境变量 +- name: set storm env and chkconfig keepnimalive/keepsupalive + shell: cd {{ install_path }}/{{ storm_version }}/bin/ && ./set_storm_env.sh + +#配置环境变量 +- name: set storm conf + shell: cd {{ install_path }}/{{ storm_version }}/conf/ && ./set_storm_conf.sh + +- name: Ansible delete {{ storm_version }}.tar.gz + file: + path: "{{ install_path }}/{{ storm_version }}.tar.gz" + state: absent + diff --git a/parcels/roles/components/storm/tasks/main.yml b/parcels/roles/components/storm/tasks/main.yml new file mode 100644 index 0000000..19987b7 --- /dev/null +++ b/parcels/roles/components/storm/tasks/main.yml @@ -0,0 +1 @@ +- include: install_storm.yml diff --git a/parcels/roles/components/storm/templates/dae-nimbus.sh.j2 b/parcels/roles/components/storm/templates/dae-nimbus.sh.j2 new file mode 100755 index 0000000..deeb9ec --- /dev/null +++ b/parcels/roles/components/storm/templates/dae-nimbus.sh.j2 @@ -0,0 +1,27 @@ +#!/bin/sh + +#修改BASE_DIR为安装的路径 +#此进程为storm集群管理进程依据集群情况，启动1-3台 + +PRO_NAME=nimbus +BASE_DIR={{ install_path }} +VERSION={{ storm_version }} +source /etc/profile + +while true ; do + NUM=`jps -l | egrep -w 'org.apache.storm.daemon.nimbus|config_value' | grep -v grep |wc -l` + if [ "${NUM}" -lt "1" ];then + nohup $BASE_DIR/$VERSION/bin/storm nimbus > /dev/null 2>&1 & + OLD_NUM=`cat $BASE_DIR/$VERSION/logs/nimbus_sum.log` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASE_DIR/$VERSION/logs/nimbus_sum.log + echo "`date "+%Y-%m-%d %H:%M:%S"` - nimbus服务启动/异常重启 - 重启次数 -> $RESTART_NUM" >> $BASE_DIR/$VERSION/nimbus_restart.log + fi + + UINUM=`jps -l | egrep -w 'org.apache.storm.ui.core|config_value' | grep -v grep |wc -l` + if [ "${UINUM}" -lt "1" ];then + nohup $BASE_DIR/$VERSION/bin/storm ui > /dev/null 2>&1 & + fi + + sleep 30 +done diff --git a/parcels/roles/components/storm/templates/dae-supervisor.sh.j2 b/parcels/roles/components/storm/templates/dae-supervisor.sh.j2 new file mode 100755 index 0000000..cb65789 --- /dev/null +++ b/parcels/roles/components/storm/templates/dae-supervisor.sh.j2 @@ -0,0 +1,21 @@ +#!/bin/sh + +#修改BASE_DIR为安装的路径 +#此进程为storm集群管理进程依据集群情况，启动1-3台 + +PRO_NAME=supervisor +BASE_DIR={{ install_path }} +VERSION={{ storm_version }} +source /etc/profile + +while true ; do + NUM=`jps -l | egrep -w 'org.apache.storm.daemon.supervisor|config_value' | grep -v grep |wc -l` + if [ "${NUM}" -lt "1" ];then + nohup $BASE_DIR/$VERSION/bin/storm supervisor > /dev/null 2>&1 & + OLD_NUM=`cat $BASE_DIR/$VERSION/logs/supervisor_sum.log` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASE_DIR/$VERSION/logs/supervisor_sum.log + echo "`date "+%Y-%m-%d %H:%M:%S"` - nimbus服务启动/异常重启 - 重启次数 -> $RESTART_NUM" >> $BASE_DIR/$VERSION/supervisor_restart.log + fi + sleep 60 +done diff --git a/parcels/roles/components/storm/templates/keepnimalive.j2 b/parcels/roles/components/storm/templates/keepnimalive.j2 new file mode 100755 index 0000000..92dd2d3 --- /dev/null +++ b/parcels/roles/components/storm/templates/keepnimalive.j2 @@ -0,0 +1,47 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 40 60 +# description: keepnimalive + +source /etc/profile +PRO_NAME=keepnimalive + +case $1 in +start) +keepnum=`ps -ef | grep dae-nimbus.sh | grep -v grep | wc -l` + +if [ $keepnum -eq "0" ];then +nohup {{ install_path }}/{{ storm_version }}/bin/dae-nimbus.sh > /dev/null 2>&1 & +fi +;; + +stop) +keeppid=`ps -ef | grep dae-nimbus.sh | grep -v grep | awk '{print $2}'` +echo "守护进程PID：$keeppid" +kill -9 $keeppid +sleep 1 +nimpid=`jps | grep nimbus | grep -v grep | awk '{print $1}'` +echo "nimbus PID：$nimpid" +kill -9 $nimpid +sleep 1 +uipid=`jps -l | grep "org.apache.storm.ui.core" | grep -v grep | awk '{print $1}'` +echo "storm-ui PID：$uipid" +kill -9 $uipid + +;; + +status) +num=`jps | grep -w nimbus | grep -v grep |wc -l` +if [ "$num" -eq "1" ];then +echo "storm-nimbus进程已启动" +else +echo "storm-nimbus进程未启动" +fi +;; + +* ) +echo "use keepnimalive [start|stop|status]" +;; +esac diff --git a/parcels/roles/components/storm/templates/keepsupalive.j2 b/parcels/roles/components/storm/templates/keepsupalive.j2 new file mode 100755 index 0000000..b9b5f5a --- /dev/null +++ b/parcels/roles/components/storm/templates/keepsupalive.j2 @@ -0,0 +1,43 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 40 60 +# description: keepsupalive + +source /etc/profile +PRO_NAME=keepsupalive + +case $1 in +start) +keepnum=`ps -ef | grep dae-supervisor.sh | grep -v grep | wc -l` + +if [ $keepnum -eq "0" ];then +nohup {{ install_path }}/{{ storm_version }}/bin/dae-supervisor.sh > /dev/null 2>&1 & +fi +;; + +stop) +keeppid=`ps -ef | grep dae-supervisor.sh | grep -v grep | awk '{print $2}'` +echo "守护进程PID：$keeppid" +kill -9 $keeppid +sleep 1 +nimpid=`jps | grep supervisor | grep -v grep | awk '{print $1}'` +echo "supervisor PID：$nimpid" +kill -9 $nimpid +;; + +status) +num=`jps | grep -w supervisor | grep -v grep |wc -l` +if [ "$num" -eq "1" ];then +echo "storm-supervisor 进程已启动" +else +echo "storm-supervisor 进程未启动" +fi +;; + +* ) +echo "use keepsupalive [start|stop|status]" +;; +esac + diff --git a/parcels/roles/components/storm/templates/set_storm_conf.sh.j2 b/parcels/roles/components/storm/templates/set_storm_conf.sh.j2 new file mode 100755 index 0000000..cc42b52 --- /dev/null +++ b/parcels/roles/components/storm/templates/set_storm_conf.sh.j2 @@ -0,0 +1,70 @@ +#!/bin/bash + +INS_DIR={{ install_path }} +PORTS={{ storm_slot_nums }} +MAX_MEM={{ storm_worker_max_mem }} +MIN_MEM={{ storm_worker_min_mem }} +VERSION={{ storm_version }} +ZK_IP={{ zookeeper_servers }} +STORM_IPS={{ install_path }}/{{ storm_version }}/conf/ip_list + +function setnimbus() { + num=`cat $STORM_IPS | wc -l` + if [ $num -gt "3" ];then + nimbus=\"`sed -n '1,1p' $STORM_IPS`\",\"`sed -n '2,2p' $STORM_IPS`\",\"`sed -n '3,3p' $STORM_IPS`\" + else + nimbus=\"`sed -n '1,1p' $STORM_IPS`\" + fi + echo -e "\n nimbus.seeds: [$nimbus]" >> $INS_DIR/$VERSION/conf/storm.yaml +} + +function setzookeeper() { + echo -e "\n storm.zookeeper.servers:" >> $INS_DIR/$VERSION/conf/storm.yaml + zkarray=(${ZK_IP//,/ }) + for zk in ${zkarray[@]} + do + address=(${zk//:/ }) + echo " - \"${address[0]}\"" >> $INS_DIR/$VERSION/conf/storm.yaml + done +} + + +function setlocaldir() { + echo -e "\n storm.local.dir: \"$INS_DIR/storm\"" >> $INS_DIR/$VERSION/conf/storm.yaml +} + + +function setslot() { + echo -e "\n supervisor.slots.ports:" >> $INS_DIR/$VERSION/conf/storm.yaml + + for i in $(seq 1 $PORTS) + do + if [ $i -lt 10 ] + then + echo " - 670$i" >> $INS_DIR/$VERSION/conf/storm.yaml + else + echo " - 67$i" >> $INS_DIR/$VERSION/conf/storm.yaml + fi + done +} + +function setothers() { + echo -e "\n ui.port: 8080" >> $INS_DIR/$VERSION/conf/storm.yaml + echo -e "\n# to nimbus" >> $INS_DIR/$VERSION/conf/storm.yaml + echo -e " nimbus.childopts: \"-Xmx1024m\"" >> $INS_DIR/$VERSION/conf/storm.yaml + echo -e "\n# to supervisor" >> $INS_DIR/$VERSION/conf/storm.yaml + echo -e " supervisor.childopts: \"-Xmx1024m\"" >> $INS_DIR/$VERSION/conf/storm.yaml + echo -e "\n# to worker" >> $INS_DIR/$VERSION/conf/storm.yaml + echo -e ' worker.childopts: "-Xmx'$MAX_MEM'm -Xms'$MIN_MEM'm -XX:+UnlockExperimentalVMOptions -XX:G1MaxNewSizePercent=80 -XX:ParallelGCThreads=12"' >> $INS_DIR/$VERSION/conf/storm.yaml + echo -e "\n supervisor.heartbeat.frequency.secs: 10" >> $INS_DIR/$VERSION/conf/storm.yaml + echo -e "\n task.heartbeat.frequency.secs: 6" >> $INS_DIR/$VERSION/conf/storm.yaml + echo -e "\n storm.messaging.netty.server_worker_threads: 4" >> $INS_DIR/$VERSION/conf/storm.yaml + echo -e "\n storm.messaging.netty.client_worker_threads: 4" >> $INS_DIR/$VERSION/conf/storm.yaml +} + + +setzookeeper +setnimbus +setlocaldir +setslot +setothers diff --git a/parcels/roles/components/storm/templates/set_storm_env.sh.j2 b/parcels/roles/components/storm/templates/set_storm_env.sh.j2 new file mode 100755 index 0000000..ce28c15 --- /dev/null +++ b/parcels/roles/components/storm/templates/set_storm_env.sh.j2 @@ -0,0 +1,23 @@ +#!/bin/bash + +num=`cat /etc/profile | grep storm | wc -l` +if [ $num -eq "0" ];then + echo -e "\n#storm" >> /etc/profile + echo -e "export STORM_HOME={{ install_path }}/{{ storm_version }}" >> /etc/profile + echo -e "export PATH=\$STORM_HOME/bin:\$PATH" >> /etc/profile + source /etc/profile +fi + + keeppath='/etc/init.d/keepnimalive' + if [ -x $keeppath ];then + chkconfig --add keepnimalive + chkconfig keepnimalive on + service keepnimalive start + fi + + keeppath='/etc/init.d/keepsupalive' + if [ -x $keeppath ];then + chkconfig --add keepsupalive + chkconfig keepsupalive on + service keepsupalive start + fi diff --git a/parcels/roles/components/storm/templates/storm.yaml.j2 b/parcels/roles/components/storm/templates/storm.yaml.j2 new file mode 100644 index 0000000..8a86057 --- /dev/null +++ b/parcels/roles/components/storm/templates/storm.yaml.j2 @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +########### These MUST be filled in for a storm configuration + + storm.zookeeper.servers: +{% for dev_info in devid_host_list %} + - "{{dev_info.host}}" +{% endfor %} + + nimbus.seeds: ["{{ nimbus_list }}"] + + storm.local.dir: "{{ install_path }}/storm" + + supervisor.slots.ports: + - 6701 + - 6702 + - 6703 + - 6704 + - 6705 + - 6706 + - 6707 + - 6708 + - 6709 + - 6710 + + ui.port: 8080 + +# to nimbus + nimbus.childopts: "-Xmx1024m" + +# to supervisor + supervisor.childopts: "-Xmx1024m" + +# to worker + worker.childopts: "-Xmx1024m" + + supervisor.heartbeat.frequency.secs: 10 + + task.heartbeat.frequency.secs: 6 + +#指定netty服务器工作线程数量 + storm.messaging.netty.server_worker_threads: 4 + +#指定netty客户端工作线程数量 + storm.messaging.netty.client_worker_threads: 4 diff --git a/parcels/roles/components/storm/templates/unload_storm.sh.j2 b/parcels/roles/components/storm/templates/unload_storm.sh.j2 new file mode 100755 index 0000000..422f08f --- /dev/null +++ b/parcels/roles/components/storm/templates/unload_storm.sh.j2 @@ -0,0 +1,39 @@ +#!/bin/bash +source /etc/profile + +function killService(){ +keeppath='/etc/init.d/keepnimalive' +if [ -x $keeppath ];then +service keepnimalive stop +chkconfig keepnimalive off +systemctl daemon-reload +rm -rf /etc/init.d/keepnimalive +fi + +keeppath='/etc/init.d/keepsupalive' +if [ -x $keeppath ];then +service keepsupalive stop +chkconfig keepsupalive off +systemctl daemon-reload +rm -rf /etc/init.d/keepsupalive +fi +} + +function killPid(){ +livenum=`jps -l | egrep -w "org.apache.storm.daemon.nimbus" | grep -v grep |wc -l` +if [ $livenum -ne 0 ];then +keeppid=`jps -l |egrep -w "org.apache.storm.daemon.nimbus" | awk '{print $1}'` +kill -9 $keeppid +fi + +livenum=`jps -l | egrep -w "org.apache.storm.daemon.supervisor" | grep -v grep |wc -l` +if [ $livenum -ne 0 ];then +keeppid=`jps -l |egrep -w "org.apache.storm.daemon.supervisor" | awk '{print $1}'` +kill -9 $keeppid +fi +} + +killService +sleep 5 +killPid +rm -rf {{ install_path }}/{{ storm_version }} diff --git a/parcels/roles/components/storm/templates/worker.xml.j2 b/parcels/roles/components/storm/templates/worker.xml.j2 new file mode 100644 index 0000000..2797c0d --- /dev/null +++ b/parcels/roles/components/storm/templates/worker.xml.j2 @@ -0,0 +1,92 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<configuration monitorInterval="60"> +<properties> + <property name="pattern">%d{yyyy-MM-dd HH:mm:ss.SSS} ${sys:storm.id}-${sys:worker.port} %contextName %c{1.} [%p] %msg%n</property> + <property name="patternNoTime">%msg%n</property> + <property name="patternMetrics">%d %-8r %m%n</property> +</properties> +<appenders> + <RollingFile name="A1" + fileName="{{ install_path }}/topologylogs/${sys:logfile.name}" + filePattern="{{ install_path }}/topologylogs/old/logs/${sys:logfile.name}.%i.gz"> + <PatternLayout> + <pattern>${pattern}</pattern> + </PatternLayout> + <Policies> + <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + </Policies> + <DefaultRolloverStrategy max="9"/> + </RollingFile> + <RollingFile name="STDOUT" + fileName="{{ install_path }}/topologylogs/${sys:logfile.name}.out" + filePattern="{{ install_path }}/topologylogs/old/outs/${sys:logfile.name}.out.%i.gz"> + <PatternLayout> + <pattern>${patternNoTime}</pattern> + </PatternLayout> + <Policies> + <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + </Policies> + <DefaultRolloverStrategy max="4"/> + </RollingFile> + <RollingFile name="STDERR" + fileName="{{ install_path }}/topologylogs/${sys:logfile.name}.err" + filePattern="{{ install_path }}/topologylogs/old/errs/${sys:logfile.name}.err.%i.gz"> + <PatternLayout> + <pattern>${patternNoTime}</pattern> + </PatternLayout> + <Policies> + <SizeBasedTriggeringPolicy size="100 MB"/> <!-- Or every 100 MB --> + </Policies> + <DefaultRolloverStrategy max="4"/> + </RollingFile> + <RollingFile name="METRICS" + fileName="{{ install_path }}/topologylogs/${sys:logfile.name}.metrics" + filePattern="{{ install_path }}/topologylogs/old/metrics/${sys:logfile.name}.metrics.%i.gz"> + <PatternLayout> + <pattern>${patternMetrics}</pattern> + </PatternLayout> + <Policies> + <SizeBasedTriggeringPolicy size="2 MB"/> + </Policies> + <DefaultRolloverStrategy max="9"/> + </RollingFile> + <Syslog name="syslog" format="RFC5424" charset="UTF-8" host="localhost" port="514" + protocol="UDP" appName="[${sys:storm.id}:${sys:worker.port}]" mdcId="mdc" includeMDC="true" + facility="LOCAL5" enterpriseNumber="18060" newLine="true" exceptionPattern="%rEx{full}" + messageId="[${sys:user.name}:${sys:logging.sensitivity}]" id="storm" immediateFail="true" immediateFlush="true"/> +</appenders> +<loggers> + <root level="error"> <!-- We log everything --> + <appender-ref ref="A1"/> + <appender-ref ref="syslog"/> + </root> + <Logger name="org.apache.storm.metric.LoggingMetricsConsumer" level="info" additivity="false"> + <appender-ref ref="METRICS"/> + </Logger> + <Logger name="STDERR" level="info"> + <appender-ref ref="STDERR"/> + <appender-ref ref="syslog"/> + </Logger> + <Logger name="STDOUT" level="info"> + <appender-ref ref="STDOUT"/> + <appender-ref ref="syslog"/> + </Logger> +</loggers> +</configuration> + diff --git a/parcels/roles/components/storm/vars/main.yml b/parcels/roles/components/storm/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/components/storm/vars/main.yml diff --git a/parcels/roles/components/test/tasks/install_zk.yml b/parcels/roles/components/test/tasks/install_zk.yml new file mode 100644 index 0000000..8c32a50 --- /dev/null +++ b/parcels/roles/components/test/tasks/install_zk.yml @@ -0,0 +1,10 @@ +- name: check jdk 1.8_73 + shell: if [ `expr {{groups.zookeeper|length}} % 2` -eq 0 ];then echo 1 ;else echo 0 ;fi + ignore_errors: false + register: jdk_out + +- name: To terminate execution + fail: + msg: "JDK 未安装 请检查" + when: jdk_out.stdout != '1' + diff --git a/parcels/roles/components/test/tasks/main.yml b/parcels/roles/components/test/tasks/main.yml new file mode 100644 index 0000000..a9675ee --- /dev/null +++ b/parcels/roles/components/test/tasks/main.yml @@ -0,0 +1 @@ +- include: install_zk.yml diff --git a/parcels/roles/components/test/tasks/unload_zk.sh.j2 b/parcels/roles/components/test/tasks/unload_zk.sh.j2 new file mode 100755 index 0000000..730720d --- /dev/null +++ b/parcels/roles/components/test/tasks/unload_zk.sh.j2 @@ -0,0 +1,23 @@ +#!/bin/bash +source /etc/profile + +service keepzkalive stop +sleep 5 + +livenum=`jps -l | grep "org.apache.zookeeper.server.quorum.QuorumPeerMain" | wc -l` +if [ $livenum -ne 0 ];then +keeppid=`jps -l |grep zookeeper | awk '{print $1}'` +kill -9 $keeppid + +rm -rf {{ install_path }}/{{ zookeeper_version }} +rm -rf {{ data_path }}/{{ zookeeper_version }} +chkconfig keepzkalive off +systemctl daemon-reload +rm -rf /etc/init.d/keepzkalive +else +rm -rf {{ install_path }}/{{ zookeeper_version }} +rm -rf {{ data_path }}/{{ zookeeper_version }} +chkconfig keepzkalive off +systemctl daemon-reload +rm -rf /etc/init.d/keepzkalive +fi diff --git a/parcels/roles/components/test/templates/change_myid.sh.j2 b/parcels/roles/components/test/templates/change_myid.sh.j2 new file mode 100644 index 0000000..add9ae1 --- /dev/null +++ b/parcels/roles/components/test/templates/change_myid.sh.j2 @@ -0,0 +1,14 @@ +#!/bin/bash + +id={{ zk_start_myid }} + +for i in `echo "{{ ansible_play_hosts }}" | grep -E -o "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+"` +do + ip=`echo $i | grep -E -o "[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"` + echo 'server.'$id'='$ip':2888:3888' >> {{ install_path }}/{{ zookeeper_version }}/conf/zoo.cfg + if [[ $ip == {{ inventory_hostname }} ]];then + echo $id > {{ install_path }}/{{ zookeeper_version }}/data/myid + fi + ((id++)) +done + diff --git a/parcels/roles/components/test/templates/dae-zookeeper.sh.j2 b/parcels/roles/components/test/templates/dae-zookeeper.sh.j2 new file mode 100644 index 0000000..98f4629 --- /dev/null +++ b/parcels/roles/components/test/templates/dae-zookeeper.sh.j2 @@ -0,0 +1,21 @@ +#!/bin/sh + +#进程名称 +PRO_NAME=QuorumPeerMain +#安装路径 +BASE_DIR={{ install_path }} +VERSION={{ zookeeper_version }} +source /etc/profile + + +while true ; do + NUM=`$BASE_DIR/$VERSION/bin/zkServer.sh status | egrep 'leader|follower' | wc -l` + if [ "${NUM}" -lt "1" ];then + $BASE_DIR/$VERSION/bin/zkServer.sh start + OLD_NUM=`cat $BASE_DIR/$VERSION/logs/restart_sum.log` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASE_DIR/$VERSION/logs/restart_sum.log + echo "`date "+%Y-%m-%d %H:%M:%S"` - zookeeper服务启动/异常重启 - 重启次数 -> $RESTART_NUM" >> $BASE_DIR/$VERSION/restart.log + fi + sleep 60 +done diff --git a/parcels/roles/components/test/templates/java.env.j2 b/parcels/roles/components/test/templates/java.env.j2 new file mode 100644 index 0000000..0f3019c --- /dev/null +++ b/parcels/roles/components/test/templates/java.env.j2 @@ -0,0 +1,6 @@ +#!/bin/sh + +export JAVA_HOME=/usr/lib/jvm/jdk1.8.0_73 +# heap size MUST be modified according to cluster environment + +export JVMFLAGS="-Xmx{{ zookeeper_max_mem }}m -Xms1024m $JVMFLAGS" diff --git a/parcels/roles/components/test/templates/keepzkalive.j2 b/parcels/roles/components/test/templates/keepzkalive.j2 new file mode 100644 index 0000000..65de403 --- /dev/null +++ b/parcels/roles/components/test/templates/keepzkalive.j2 @@ -0,0 +1,34 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 20 80 +# description: keepzkalive +source /etc/profile +PRO_NAME=keepzklive + +case $1 in +start) +keepnum=`ps -ef | grep dae-zookeeper.sh | grep -v grep | wc -l` + +if [ $keepnum -eq "0" ];then +nohup {{ install_path }}/{{ zookeeper_version }}/bin/dae-zookeeper.sh /home/bigdata > /dev/null 2>&1 & +fi +;; + +stop) +keeppid=`ps -ef | grep dae-zookeeper.sh | grep -v grep | awk '{print $2}'` +echo "守护进程PID：$keeppid" +kill -9 $keeppid +sleep 1 +sh {{ install_path }}/{{ zookeeper_version }}/bin/zkServer.sh stop +;; + +status) +sh {{ install_path }}/{{ zookeeper_version }}/bin/zkServer.sh status +;; + +* ) +echo "use keepzklive [start|stop|status]" +;; +esac diff --git a/parcels/roles/components/test/templates/myid.j2 b/parcels/roles/components/test/templates/myid.j2 new file mode 100644 index 0000000..43d4730 --- /dev/null +++ b/parcels/roles/components/test/templates/myid.j2 @@ -0,0 +1,5 @@ +{% for dev_info in zk_host_list %} +{% if inventory_hostname == dev_info.host %} +{{dev_info.myid}} +{% endif %} +{% endfor %} diff --git a/parcels/roles/components/test/templates/set_zk_env.sh.j2 b/parcels/roles/components/test/templates/set_zk_env.sh.j2 new file mode 100644 index 0000000..305ccd2 --- /dev/null +++ b/parcels/roles/components/test/templates/set_zk_env.sh.j2 @@ -0,0 +1,17 @@ +#!/bin/bash + +num=`cat /etc/profile | grep zookeeper | wc -l` + +if [ $num -eq "0" ];then + echo -e "\n#zookeeper" >> /etc/profile + echo -e "export ZOOKEEPER_HOME={{ install_path }}/{{ zookeeper_version }}" >> /etc/profile + echo -e "export PATH=\$ZOOKEEPER_HOME/bin:\$PATH" >> /etc/profile + source /etc/profile +fi + +keeppsth='/etc/init.d/keepzkalive' +if [ -x $keeppsth ];then + chkconfig --add keepzkalive + chkconfig keepzkalive on + service keepzkalive start +fi diff --git a/parcels/roles/components/test/templates/start_exporter.sh.j2 b/parcels/roles/components/test/templates/start_exporter.sh.j2 new file mode 100644 index 0000000..39714df --- /dev/null +++ b/parcels/roles/components/test/templates/start_exporter.sh.j2 @@ -0,0 +1,7 @@ +#!/bin/bash +docker rm -f zookeeper_exporter + +sleep 3 + +docker run -d -p 9902:9114 --name zookeeper_exporter --restart always zookeeper_exporter:v1.0 {{ inventory_hostname }}:2181 + diff --git a/parcels/roles/components/test/templates/zoo.cfg.j2 b/parcels/roles/components/test/templates/zoo.cfg.j2 new file mode 100644 index 0000000..727056c --- /dev/null +++ b/parcels/roles/components/test/templates/zoo.cfg.j2 @@ -0,0 +1,31 @@ +# The number of milliseconds of each tick +tickTime=9000 +# The number of ticks that the initial +# synchronization phase can take +initLimit=10 +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit=5 +# the directory where the snapshot is stored. +# do not use /tmp for storage, /tmp here is just +# example sakes. +dataDir={{ data_path }}/{{ zookeeper_version }}/data +# the port at which the clients will connect +clientPort={{ zookeeper_port }} +# the maximum number of client connections. +# increase this if you need to handle more clients +maxClientCnxns=1000 +# +# Be sure to read the maintenance section of the +# administrator guide before turning on autopurge. +# +# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance +# +# The number of snapshots to retain in dataDir +autopurge.snapRetainCount=5 +# Purge task interval in hours +# Set to "0" to disable auto purge feature +autopurge.purgeInterval=1 +dataLogDir={{ install_path }}/{{ zookeeper_version }}/logs +# +#the servers diff --git a/parcels/roles/components/test/vars/main.yml b/parcels/roles/components/test/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/components/test/vars/main.yml diff --git a/parcels/roles/components/zookeeper/tasks/install_zk.yml b/parcels/roles/components/zookeeper/tasks/install_zk.yml new file mode 100644 index 0000000..fbd7691 --- /dev/null +++ b/parcels/roles/components/zookeeper/tasks/install_zk.yml @@ -0,0 +1,175 @@ +- name: check zookeeper nodes is odd number + shell: if [ `expr {{groups.zookeeper|length}} % 2` -eq 0 ];then echo 1 ;else echo 0 ;fi + ignore_errors: false + register: nums_out + run_once: true + delegate_to: 127.0.0.1 + +- name: To terminate execution + fail: + msg: "zookeeper 节点配置数量必须为奇数台，请修改。" + when: nums_out.stdout != '0' + run_once: true + delegate_to: 127.0.0.1 + +- name: check jdk 1.8_73 + shell: source /etc/profile && java -version 2>&1 | grep 1.8.0_73 | wc -l + ignore_errors: false + register: jdk_out + +- name: To terminate execution + fail: + msg: "JDK 未安装 请检查" + when: jdk_out.stdout != '2' + +- name: create zookeeper package, path:{{ install_path }} + file: + state: directory + path: '{{ install_path }}' + +- block: + - name: Check if the zookeeper service already exists + shell: source /etc/profile && lsof -i:{{ zookeeper_port }} | grep `jps -l | grep "org.apache.zookeeper.server.quorum.QuorumPeerMain" | grep -v grep |awk '{print $1}'` |grep LISTEN | wc -l + register: check_out + - name: copy unload_zk.sh to {{ install_path }}/ + template: + src: 'unload_zk.sh.j2' + dest: '{{ install_path }}/unload_zk.sh' + force: true + mode: 0755 + when: check_out.stdout >= '1' + - name: unload zookeeper + shell: cd {{ install_path }} && sh unload_zk.sh + when: check_out.stdout >= '1' + - name: Ansible delete {{ install_path }}/unload_zk.sh + file: + path: "{{ install_path }}/unload_zk.sh" + state: absent + - name: Check if the zookeeper service already exists + shell: source /etc/profile && jps -l | grep "org.apache.zookeeper.server.quorum.QuorumPeerMain" | grep -v grep |awk '{print $1}' | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "卸载失败，组件可能非本安装部署，请手动卸载后继续安装" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "yes" + +- block: + - name: Check if the zookeeper service already exists + shell: source /etc/profile && lsof -i:{{ zookeeper_port }} | grep `jps -l | grep "org.apache.zookeeper.server.quorum.QuorumPeerMain" | grep -v grep |awk '{print $1}'` |grep LISTEN | wc -l + register: check_out + - name: To terminate execution + fail: + msg: "检测到目标集群内已有Zookeeper请检查IP是否正确或停止/卸载已启动的组件" + run_once: true + delegate_to: 127.0.0.1 + when: check_out.stdout >= '1' + when: (allowed_unload) == "no" + +- name: create zookeeper data package, path:{{ data_path }}/{{ zookeeper_version }} + file: + state: directory + path: '{{ data_path }}/{{ zookeeper_version }}' + +#复制tar到目标服务器 +- name: copy {{ zookeeper_version }}.tar.gz to {{ install_path }}/ + copy: + src: '{{ package_path }}/{{ zookeeper_version }}.tar.gz' + dest: '{{ install_path }}/' + force: true + backup: yes + +#解压tar +- name: unpack {{ install_path }}/{{ zookeeper_version }}.tar.gz + unarchive: + src: '{{ zookeeper_version }}.tar.gz' + dest: '{{ install_path }}/' + +#拷贝配置文件 +- name: copy zoo.cfg to {{ install_path }}/{{ zookeeper_version }}/conf/ + template: + src: 'zoo.cfg.j2' + dest: '{{ install_path }}/{{ zookeeper_version }}/conf/zoo.cfg' + force: true + backup: yes + +- name: copy start_exporter.sh to {{ install_path }}/{{ zookeeper_version }}/monitor/ + template: + src: 'start_exporter.sh.j2' + dest: '{{ install_path }}/{{ zookeeper_version }}/monitor/start_exporter.sh' + mode: 0755 + force: true + +- name: copy dae-zookeeper.sh to {{ install_path }}/{{ zookeeper_version }}/bin/ + template: + src: 'dae-zookeeper.sh.j2' + dest: '{{ install_path }}/{{ zookeeper_version }}/bin/dae-zookeeper.sh' + mode: 0755 + force: true + +- name: copy keepzkalive to /etc/init.d/ + template: + src: 'keepzkalive.j2' + dest: '/etc/init.d/keepzkalive' + mode: 0755 + force: true + +- name: copy set_zk_env.sh to {{ install_path }}/{{ zookeeper_version }}/bin/ + template: + src: 'set_zk_env.sh.j2' + dest: '{{ install_path }}/{{ zookeeper_version }}/bin/set_zk_env.sh' + mode: 0755 + force: true + backup: yes + +- name: copy java.env to {{ install_path }}/{{ zookeeper_version }}/conf/ + template: + src: 'java.env.j2' + dest: '{{ install_path }}/{{ zookeeper_version }}/conf/java.env' + force: true + backup: yes + +- name: copy change_myid.sh to {{ install_path }}/{{ zookeeper_version }}/bin/ + template: + src: 'change_myid.sh.j2' + dest: '{{ install_path }}/{{ zookeeper_version }}/bin/change_myid.sh' + mode: 0755 + force: true + backup: yes + +- name: set zookeeper env and chkconfig keepzkalive + shell: cd {{ install_path }}/{{ zookeeper_version }}/bin/ && ./set_zk_env.sh && ./change_myid.sh + +- name: Ansible delete {{ zookeeper_version }}.tar.gz + file: + path: "{{ install_path }}/{{ zookeeper_version }}.tar.gz" + state: absent + +- name: load zookeeper_expoter + docker_image: + name: 'zookeeper_exporter' + tag: 'v1.0' + load_path: '{{ install_path }}/{{ zookeeper_version }}/monitor/zookeeper_exporter.tar.gz' + source: load + force_tag: yes + force_source: yes + +- name: start keepzkalive service + shell: source /etc/profile && sh /etc/init.d/keepzkalive 'start' + +- name: waiting zookeeper start sleep 10 s + shell: sleep 10 + +- name: check zookeeper status + shell: source /etc/profile && zkServer.sh status | egrep 'leader|follower' | wc -l + register: zkstatus_out + +- name: start keepzkalive service again + shell: source /etc/profile && sh /etc/init.d/keepzkalive 'start' + when: zkstatus_out.stdout != '1' + +- name: start zookeeper_exporter + shell: cd {{ install_path }}/{{ zookeeper_version }}/monitor/ && ./start_exporter.sh + diff --git a/parcels/roles/components/zookeeper/tasks/main.yml b/parcels/roles/components/zookeeper/tasks/main.yml new file mode 100644 index 0000000..a9675ee --- /dev/null +++ b/parcels/roles/components/zookeeper/tasks/main.yml @@ -0,0 +1 @@ +- include: install_zk.yml diff --git a/parcels/roles/components/zookeeper/templates/change_myid.sh.j2 b/parcels/roles/components/zookeeper/templates/change_myid.sh.j2 new file mode 100755 index 0000000..add9ae1 --- /dev/null +++ b/parcels/roles/components/zookeeper/templates/change_myid.sh.j2 @@ -0,0 +1,14 @@ +#!/bin/bash + +id={{ zk_start_myid }} + +for i in `echo "{{ ansible_play_hosts }}" | grep -E -o "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+"` +do + ip=`echo $i | grep -E -o "[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+"` + echo 'server.'$id'='$ip':2888:3888' >> {{ install_path }}/{{ zookeeper_version }}/conf/zoo.cfg + if [[ $ip == {{ inventory_hostname }} ]];then + echo $id > {{ install_path }}/{{ zookeeper_version }}/data/myid + fi + ((id++)) +done + diff --git a/parcels/roles/components/zookeeper/templates/dae-zookeeper.sh.j2 b/parcels/roles/components/zookeeper/templates/dae-zookeeper.sh.j2 new file mode 100755 index 0000000..6357bb5 --- /dev/null +++ b/parcels/roles/components/zookeeper/templates/dae-zookeeper.sh.j2 @@ -0,0 +1,21 @@ +#!/bin/bash + +#进程名称 +PRO_NAME=QuorumPeerMain +#安装路径 +BASE_DIR={{ install_path }} +VERSION={{ zookeeper_version }} +source /etc/profile + + +while true ; do + NUM=`$BASE_DIR/$VERSION/bin/zkServer.sh status | egrep 'leader|follower' | wc -l` + if [ "${NUM}" -lt "1" ];then + $BASE_DIR/$VERSION/bin/zkServer.sh start + OLD_NUM=`cat $BASE_DIR/$VERSION/logs/restart_sum.log` + RESTART_NUM=`expr $OLD_NUM + 1` + echo $RESTART_NUM > $BASE_DIR/$VERSION/logs/restart_sum.log + echo "`date "+%Y-%m-%d %H:%M:%S"` - zookeeper服务启动/异常重启 - 重启次数 -> $RESTART_NUM" >> $BASE_DIR/$VERSION/restart.log + fi + sleep 60 +done diff --git a/parcels/roles/components/zookeeper/templates/java.env.j2 b/parcels/roles/components/zookeeper/templates/java.env.j2 new file mode 100644 index 0000000..0f3019c --- /dev/null +++ b/parcels/roles/components/zookeeper/templates/java.env.j2 @@ -0,0 +1,6 @@ +#!/bin/sh + +export JAVA_HOME=/usr/lib/jvm/jdk1.8.0_73 +# heap size MUST be modified according to cluster environment + +export JVMFLAGS="-Xmx{{ zookeeper_max_mem }}m -Xms1024m $JVMFLAGS" diff --git a/parcels/roles/components/zookeeper/templates/keepzkalive.j2 b/parcels/roles/components/zookeeper/templates/keepzkalive.j2 new file mode 100644 index 0000000..65de403 --- /dev/null +++ b/parcels/roles/components/zookeeper/templates/keepzkalive.j2 @@ -0,0 +1,34 @@ +#!/bin/bash +# +# netconsole This loads the netconsole module with the configured parameters. +# +# chkconfig:123456 20 80 +# description: keepzkalive +source /etc/profile +PRO_NAME=keepzklive + +case $1 in +start) +keepnum=`ps -ef | grep dae-zookeeper.sh | grep -v grep | wc -l` + +if [ $keepnum -eq "0" ];then +nohup {{ install_path }}/{{ zookeeper_version }}/bin/dae-zookeeper.sh /home/bigdata > /dev/null 2>&1 & +fi +;; + +stop) +keeppid=`ps -ef | grep dae-zookeeper.sh | grep -v grep | awk '{print $2}'` +echo "守护进程PID：$keeppid" +kill -9 $keeppid +sleep 1 +sh {{ install_path }}/{{ zookeeper_version }}/bin/zkServer.sh stop +;; + +status) +sh {{ install_path }}/{{ zookeeper_version }}/bin/zkServer.sh status +;; + +* ) +echo "use keepzklive [start|stop|status]" +;; +esac diff --git a/parcels/roles/components/zookeeper/templates/myid.j2 b/parcels/roles/components/zookeeper/templates/myid.j2 new file mode 100644 index 0000000..43d4730 --- /dev/null +++ b/parcels/roles/components/zookeeper/templates/myid.j2 @@ -0,0 +1,5 @@ +{% for dev_info in zk_host_list %} +{% if inventory_hostname == dev_info.host %} +{{dev_info.myid}} +{% endif %} +{% endfor %} diff --git a/parcels/roles/components/zookeeper/templates/set_zk_env.sh.j2 b/parcels/roles/components/zookeeper/templates/set_zk_env.sh.j2 new file mode 100755 index 0000000..305ccd2 --- /dev/null +++ b/parcels/roles/components/zookeeper/templates/set_zk_env.sh.j2 @@ -0,0 +1,17 @@ +#!/bin/bash + +num=`cat /etc/profile | grep zookeeper | wc -l` + +if [ $num -eq "0" ];then + echo -e "\n#zookeeper" >> /etc/profile + echo -e "export ZOOKEEPER_HOME={{ install_path }}/{{ zookeeper_version }}" >> /etc/profile + echo -e "export PATH=\$ZOOKEEPER_HOME/bin:\$PATH" >> /etc/profile + source /etc/profile +fi + +keeppsth='/etc/init.d/keepzkalive' +if [ -x $keeppsth ];then + chkconfig --add keepzkalive + chkconfig keepzkalive on + service keepzkalive start +fi diff --git a/parcels/roles/components/zookeeper/templates/start_exporter.sh.j2 b/parcels/roles/components/zookeeper/templates/start_exporter.sh.j2 new file mode 100755 index 0000000..39714df --- /dev/null +++ b/parcels/roles/components/zookeeper/templates/start_exporter.sh.j2 @@ -0,0 +1,7 @@ +#!/bin/bash +docker rm -f zookeeper_exporter + +sleep 3 + +docker run -d -p 9902:9114 --name zookeeper_exporter --restart always zookeeper_exporter:v1.0 {{ inventory_hostname }}:2181 + diff --git a/parcels/roles/components/zookeeper/templates/unload_zk.sh.j2 b/parcels/roles/components/zookeeper/templates/unload_zk.sh.j2 new file mode 100755 index 0000000..86548fb --- /dev/null +++ b/parcels/roles/components/zookeeper/templates/unload_zk.sh.j2 @@ -0,0 +1,21 @@ +#!/bin/bash +source /etc/profile + +service keepzkalive stop +sleep 5 + +livenum=`jps -l | grep "org.apache.zookeeper.server.quorum.QuorumPeerMain" | wc -l` +if [ $livenum -ne 0 ];then +keeppid=`jps -l |grep zookeeper | awk '{print $1}'` +kill -9 $keeppid + +rm -rf {{ install_path }}/{{ zookeeper_version }} +rm -rf {{ data_path }}/{{ zookeeper_version }} +chkconfig keepzkalive off +rm -rf /etc/init.d/keepzkalive +else +rm -rf {{ install_path }}/{{ zookeeper_version }} +rm -rf {{ data_path }}/{{ zookeeper_version }} +chkconfig keepzkalive off +rm -rf /etc/init.d/keepzkalive +fi diff --git a/parcels/roles/components/zookeeper/templates/zoo.cfg.j2 b/parcels/roles/components/zookeeper/templates/zoo.cfg.j2 new file mode 100644 index 0000000..727056c --- /dev/null +++ b/parcels/roles/components/zookeeper/templates/zoo.cfg.j2 @@ -0,0 +1,31 @@ +# The number of milliseconds of each tick +tickTime=9000 +# The number of ticks that the initial +# synchronization phase can take +initLimit=10 +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit=5 +# the directory where the snapshot is stored. +# do not use /tmp for storage, /tmp here is just +# example sakes. +dataDir={{ data_path }}/{{ zookeeper_version }}/data +# the port at which the clients will connect +clientPort={{ zookeeper_port }} +# the maximum number of client connections. +# increase this if you need to handle more clients +maxClientCnxns=1000 +# +# Be sure to read the maintenance section of the +# administrator guide before turning on autopurge. +# +# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance +# +# The number of snapshots to retain in dataDir +autopurge.snapRetainCount=5 +# Purge task interval in hours +# Set to "0" to disable auto purge feature +autopurge.purgeInterval=1 +dataLogDir={{ install_path }}/{{ zookeeper_version }}/logs +# +#the servers diff --git a/parcels/roles/components/zookeeper/vars/main.yml b/parcels/roles/components/zookeeper/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/components/zookeeper/vars/main.yml diff --git a/parcels/roles/dependencies/check-env/tasks/check_env.yml b/parcels/roles/dependencies/check-env/tasks/check_env.yml new file mode 100644 index 0000000..7f57b13 --- /dev/null +++ b/parcels/roles/dependencies/check-env/tasks/check_env.yml @@ -0,0 +1,68 @@ +- name: check firewall + shell: systemctl status firewalld | grep Active | grep dead | wc -l + ignore_errors: false + register: firewall_out + +- name: stop firewall + shell: systemctl stop firewalld && systemctl disable firewalld + ignore_errors: false + when: firewall_out.stdout != '1' + +- name: has stopped, check the firewall again + shell: systemctl status firewalld | grep Active | grep dead | wc -l + ignore_errors: false + register: firewall_again + +- name: To terminate execution + fail: + msg: "执行防火墙关闭命令后无效，请检查" + when: firewall_again.stdout != '1' + +- name: check jdk 1.8_73 + shell: source /etc/profile && java -version 2>&1 | grep 1.8.0_73 | wc -l + ignore_errors: false + register: jdk_out #定义变量存储返回的结果 + +- name: To terminate execution + fail: + msg: "JDK 未安装 请检查" + when: jdk_out.stdout != '2' + +- name: check timezone + shell: "cat /etc/timezone | wc -l" + ignore_errors: false + register: timezone_out #定义变量存储返回的结果 + +- name: To terminate execution + fail: + msg: "/etc/timezone 未配置 请检查" + when: timezone_out.stdout != '1' + +#- name: set timezone +# shell: cp /usr/share/zoneinfo/{{ time_zone }} /etc/localtime |cat && echo {{ time_zone }} > /etc/timezone +# when: timezone_out.stdout != '1' +# +#- name: check timezone again +# shell: cat /etc/timezone | wc -l +# ignore_errors: false +# register: timezone_again #定义变量存储返回的结果 +# +#- name: To terminate execution +# fail: +# msg: "服务器timezone文件为空，请检查" +# when: timezone_again.stdout != '1' + +#- name: check CPU support sse4_2 +# shell: lscpu |grep "sse4_2" |wc -l +# ignore_errors: false +# register: cpu_out #定义变量存储返回的结果 +# +#- name: To terminate execution +# fail: +# msg: "服务器CPU不支持 sse4_2 指令集，无法安装clickhouse组件" +# when: cpu_out.stdout != '1' + +#- debug: +# msg: " the enviroment is correct" +# when: firewall_out.stdout == '1' and jdk_out.stdout == '2' and timezone_out.stdout == '1' + diff --git a/parcels/roles/dependencies/check-env/tasks/main.yml b/parcels/roles/dependencies/check-env/tasks/main.yml new file mode 100644 index 0000000..8fa506f --- /dev/null +++ b/parcels/roles/dependencies/check-env/tasks/main.yml @@ -0,0 +1 @@ +- include: check_env.yml diff --git a/parcels/roles/dependencies/check-env/vars/main.yml b/parcels/roles/dependencies/check-env/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/dependencies/check-env/vars/main.yml diff --git a/parcels/roles/dependencies/environments/install_ansible/ansible/PyYAML-3.10-11.el7.x86_64.rpm b/parcels/roles/dependencies/environments/install_ansible/ansible/PyYAML-3.10-11.el7.x86_64.rpm Binary files differnew file mode 100644 index 0000000..6fe88a9 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_ansible/ansible/PyYAML-3.10-11.el7.x86_64.rpm diff --git a/parcels/roles/dependencies/environments/install_ansible/ansible/ansible-2.9.2-1.el7.noarch.rpm b/parcels/roles/dependencies/environments/install_ansible/ansible/ansible-2.9.2-1.el7.noarch.rpm Binary files differnew file mode 100644 index 0000000..b592e69 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_ansible/ansible/ansible-2.9.2-1.el7.noarch.rpm diff --git a/parcels/roles/dependencies/environments/install_ansible/ansible/libyaml-0.1.4-11.el7_0.x86_64.rpm b/parcels/roles/dependencies/environments/install_ansible/ansible/libyaml-0.1.4-11.el7_0.x86_64.rpm Binary files differnew file mode 100644 index 0000000..cb634d3 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_ansible/ansible/libyaml-0.1.4-11.el7_0.x86_64.rpm diff --git a/parcels/roles/dependencies/environments/install_ansible/ansible/python-babel-0.9.6-8.el7.noarch.rpm b/parcels/roles/dependencies/environments/install_ansible/ansible/python-babel-0.9.6-8.el7.noarch.rpm Binary files differnew file mode 100644 index 0000000..d481181 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_ansible/ansible/python-babel-0.9.6-8.el7.noarch.rpm diff --git a/parcels/roles/dependencies/environments/install_ansible/ansible/python-httplib2-0.9.2-1.el7.noarch.rpm b/parcels/roles/dependencies/environments/install_ansible/ansible/python-httplib2-0.9.2-1.el7.noarch.rpm Binary files differnew file mode 100644 index 0000000..65c0763 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_ansible/ansible/python-httplib2-0.9.2-1.el7.noarch.rpm diff --git a/parcels/roles/dependencies/environments/install_ansible/ansible/python-jinja2-2.7.2-4.el7.noarch.rpm b/parcels/roles/dependencies/environments/install_ansible/ansible/python-jinja2-2.7.2-4.el7.noarch.rpm Binary files differnew file mode 100644 index 0000000..d7239c8 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_ansible/ansible/python-jinja2-2.7.2-4.el7.noarch.rpm diff --git a/parcels/roles/dependencies/environments/install_ansible/ansible/python-markupsafe-0.11-10.el7.x86_64.rpm b/parcels/roles/dependencies/environments/install_ansible/ansible/python-markupsafe-0.11-10.el7.x86_64.rpm Binary files differnew file mode 100644 index 0000000..43c8ac9 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_ansible/ansible/python-markupsafe-0.11-10.el7.x86_64.rpm diff --git a/parcels/roles/dependencies/environments/install_ansible/ansible/python-paramiko-2.1.1-9.el7.noarch.rpm b/parcels/roles/dependencies/environments/install_ansible/ansible/python-paramiko-2.1.1-9.el7.noarch.rpm Binary files differnew file mode 100644 index 0000000..8c79030 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_ansible/ansible/python-paramiko-2.1.1-9.el7.noarch.rpm diff --git a/parcels/roles/dependencies/environments/install_ansible/ansible/python2-jmespath-0.9.0-3.el7.noarch.rpm b/parcels/roles/dependencies/environments/install_ansible/ansible/python2-jmespath-0.9.0-3.el7.noarch.rpm Binary files differnew file mode 100644 index 0000000..8e09135 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_ansible/ansible/python2-jmespath-0.9.0-3.el7.noarch.rpm diff --git a/parcels/roles/dependencies/environments/install_ansible/ansible/sshpass-1.06-2.el7.x86_64.rpm b/parcels/roles/dependencies/environments/install_ansible/ansible/sshpass-1.06-2.el7.x86_64.rpm Binary files differnew file mode 100644 index 0000000..a1bc0b8 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_ansible/ansible/sshpass-1.06-2.el7.x86_64.rpm diff --git a/parcels/roles/dependencies/environments/install_ansible/epel/epel-release-7-11.noarch.rpm b/parcels/roles/dependencies/environments/install_ansible/epel/epel-release-7-11.noarch.rpm Binary files differnew file mode 100644 index 0000000..9ab7d30 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_ansible/epel/epel-release-7-11.noarch.rpm diff --git a/parcels/roles/dependencies/environments/install_ansible/install_ansible.sh b/parcels/roles/dependencies/environments/install_ansible/install_ansible.sh new file mode 100755 index 0000000..3d8a420 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_ansible/install_ansible.sh @@ -0,0 +1,5 @@ +path=`pwd` + +rpm -ivh $path/epel/*.rpm --force --nodeps +rpm -ivh $path/ansible/*.rpm --force --nodeps + diff --git a/parcels/roles/dependencies/environments/install_docker/daemon.json b/parcels/roles/dependencies/environments/install_docker/daemon.json new file mode 100644 index 0000000..924c78b --- /dev/null +++ b/parcels/roles/dependencies/environments/install_docker/daemon.json @@ -0,0 +1,9 @@ +{ + "insecure-registries": [ + "10.16.43.130:5080" + ], + "registry-mirrors":[ + "https://d8b3zdiw.mirror.aliyuncs.com" + ], + "data-root": "dockerDataDir/docker" +} diff --git a/parcels/roles/dependencies/environments/install_docker/daemon.json_bak b/parcels/roles/dependencies/environments/install_docker/daemon.json_bak new file mode 100644 index 0000000..3b762fb --- /dev/null +++ b/parcels/roles/dependencies/environments/install_docker/daemon.json_bak @@ -0,0 +1,9 @@ +{ + "insecure-registries": [ + "10.16.43.130:5080" + ], + "registry-mirrors":[ + "https://d8b3zdiw.mirror.aliyuncs.com" + ], + "data-root": "/home/docker" +} diff --git a/parcels/roles/dependencies/environments/install_docker/docker-18.09.8.tgz b/parcels/roles/dependencies/environments/install_docker/docker-18.09.8.tgz Binary files differnew file mode 100644 index 0000000..26de81a --- /dev/null +++ b/parcels/roles/dependencies/environments/install_docker/docker-18.09.8.tgz diff --git a/parcels/roles/dependencies/environments/install_docker/docker-compose/docker-compose b/parcels/roles/dependencies/environments/install_docker/docker-compose/docker-compose Binary files differnew file mode 100644 index 0000000..0060bd0 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_docker/docker-compose/docker-compose diff --git a/parcels/roles/dependencies/environments/install_docker/docker.service b/parcels/roles/dependencies/environments/install_docker/docker.service new file mode 100644 index 0000000..6e4af37 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_docker/docker.service @@ -0,0 +1,46 @@ +[Unit] +Description=Docker Application Container Engine +Documentation=https://docs.docker.com +#BindsTo=containerd.service. — 无需containerd。 +After=network-online.target firewalld.service +Wants=network-online.target + +[Service] +Type=notify +# the default is not to use systemd for cgroups because the delegate issues still +# exists and systemd currently does not support the cgroup feature set required +# for containers run by docker +ExecStart=/usr/bin/dockerd -H unix:// +ExecReload=/bin/kill -s HUP $MAINPID +TimeoutSec=0 +RestartSec=2 +Restart=always + +# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229. +# Both the old, and new location are accepted by systemd 229 and up, so using the old location +# to make them work for either version of systemd. +StartLimitBurst=3 + +# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230. +# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make +# this option work for either version of systemd. +StartLimitInterval=60s + +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNOFILE=infinity +LimitNPROC=infinity +LimitCORE=infinity + +# Comment TasksMax if your systemd version does not supports it. +# Only systemd 226 and above support this option. +TasksMax=infinity + +# set delegate yes so that systemd does not reset the cgroups of docker containers +Delegate=yes + +# kill only the docker process, not all processes in the cgroup +KillMode=process + +[Install] +WantedBy=multi-user.target
\ No newline at end of file diff --git a/parcels/roles/dependencies/environments/install_docker/docker/containerd b/parcels/roles/dependencies/environments/install_docker/docker/containerd Binary files differnew file mode 100755 index 0000000..7a67143 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_docker/docker/containerd diff --git a/parcels/roles/dependencies/environments/install_docker/docker/containerd-shim b/parcels/roles/dependencies/environments/install_docker/docker/containerd-shim Binary files differnew file mode 100755 index 0000000..4bb3d58 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_docker/docker/containerd-shim diff --git a/parcels/roles/dependencies/environments/install_docker/docker/ctr b/parcels/roles/dependencies/environments/install_docker/docker/ctr Binary files differnew file mode 100755 index 0000000..cf81207 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_docker/docker/ctr diff --git a/parcels/roles/dependencies/environments/install_docker/docker/docker b/parcels/roles/dependencies/environments/install_docker/docker/docker Binary files differnew file mode 100755 index 0000000..9487cde --- /dev/null +++ b/parcels/roles/dependencies/environments/install_docker/docker/docker diff --git a/parcels/roles/dependencies/environments/install_docker/docker/docker-init b/parcels/roles/dependencies/environments/install_docker/docker/docker-init Binary files differnew file mode 100755 index 0000000..3785e00 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_docker/docker/docker-init diff --git a/parcels/roles/dependencies/environments/install_docker/docker/docker-proxy b/parcels/roles/dependencies/environments/install_docker/docker/docker-proxy Binary files differnew file mode 100755 index 0000000..510f58c --- /dev/null +++ b/parcels/roles/dependencies/environments/install_docker/docker/docker-proxy diff --git a/parcels/roles/dependencies/environments/install_docker/docker/dockerd b/parcels/roles/dependencies/environments/install_docker/docker/dockerd Binary files differnew file mode 100755 index 0000000..307836c --- /dev/null +++ b/parcels/roles/dependencies/environments/install_docker/docker/dockerd diff --git a/parcels/roles/dependencies/environments/install_docker/docker/runc b/parcels/roles/dependencies/environments/install_docker/docker/runc Binary files differnew file mode 100755 index 0000000..39b5b47 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_docker/docker/runc diff --git a/parcels/roles/dependencies/environments/install_docker/jq-linux64 b/parcels/roles/dependencies/environments/install_docker/jq-linux64 Binary files differnew file mode 100644 index 0000000..939227e --- /dev/null +++ b/parcels/roles/dependencies/environments/install_docker/jq-linux64 diff --git a/parcels/roles/dependencies/environments/install_docker/setup_docker.sh b/parcels/roles/dependencies/environments/install_docker/setup_docker.sh new file mode 100755 index 0000000..260e9b6 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_docker/setup_docker.sh @@ -0,0 +1,22 @@ +#!/bin/bash +#安装docker并设置为开机自启动 +tar zxvf `pwd`/docker-18.09.8.tgz && +cp `pwd`/docker/* /usr/bin/ && +mkdir /etc/docker && +cp `pwd`/daemon.json /etc/docker/daemon.json && +cp `pwd`/docker.service /usr/lib/systemd/system/docker.service && +#(默认执行)安装docker-compose工具 +cp `pwd`/docker-compose/docker-compose /usr/local/bin/ && +chmod +x /usr/local/bin/docker-compose && +systemctl daemon-reload && +systemctl start docker && +systemctl enable docker + +#如果报错 container init caused \"write /proc/self/attr/keycreate: permission denied\"": unknown. +#设置selinux 执行 setenforce 0 +#或者永久关闭 执行 vi /etc/selinux/config 把其中的SELINUX=enforce改成disabled就可以了 + + +#安装jq命令 +cp `pwd`/jq-linux64 /usr/bin/jq + diff --git a/parcels/roles/dependencies/environments/install_docker/xz_docker.sh b/parcels/roles/dependencies/environments/install_docker/xz_docker.sh new file mode 100644 index 0000000..a31f962 --- /dev/null +++ b/parcels/roles/dependencies/environments/install_docker/xz_docker.sh @@ -0,0 +1,7 @@ +#!/bin/bash +systemctl stop docker && +systemctl disable docker && +#执行不成功就把下面的注释 +#yum remove docker-ce && +rm -rf /var/lib/docker && +rm -rf $(whereis docker)
\ No newline at end of file diff --git a/parcels/roles/dependencies/install-docker/file/daemon.json b/parcels/roles/dependencies/install-docker/file/daemon.json new file mode 100644 index 0000000..7e455fb --- /dev/null +++ b/parcels/roles/dependencies/install-docker/file/daemon.json @@ -0,0 +1,10 @@ +{ + "insecure-registries": [ + "10.16.43.130:5080" + ], + "registry-mirrors":[ + "https://d8b3zdiw.mirror.aliyuncs.com" + ], + "data-root": "{{ install_path }}/docker" +} + diff --git a/parcels/roles/dependencies/install-docker/file/docker-compose-Linux-x86_64 b/parcels/roles/dependencies/install-docker/file/docker-compose-Linux-x86_64 Binary files differnew file mode 100644 index 0000000..d2fcdc5 --- /dev/null +++ b/parcels/roles/dependencies/install-docker/file/docker-compose-Linux-x86_64 diff --git a/parcels/roles/dependencies/install-docker/handlers/main.yml b/parcels/roles/dependencies/install-docker/handlers/main.yml new file mode 100644 index 0000000..1907c4c --- /dev/null +++ b/parcels/roles/dependencies/install-docker/handlers/main.yml @@ -0,0 +1,4 @@ +- name: restart docker + service: + name: docker + state: restarted diff --git a/parcels/roles/dependencies/install-docker/tasks/install_docker_and_compose.yml b/parcels/roles/dependencies/install-docker/tasks/install_docker_and_compose.yml new file mode 100644 index 0000000..9b55e49 --- /dev/null +++ b/parcels/roles/dependencies/install-docker/tasks/install_docker_and_compose.yml @@ -0,0 +1,46 @@ +- name: check docker version; if docker does not exist, it will be print error;Don't Worry!!! + shell: docker -v + ignore_errors: True + register: result + +- name: create docker directory + file: + path: ~/docker + state: directory + when: result.stderr is defined and result.stderr != '' #有错误信息,说明docker服务不存在 + +- name: docker service not exist; copy docker-rpm.zip and unzip docker-rpm.zip to ~/docker/docker-rpm/ + unarchive: src={{ package_path }}/docker-rpm.zip dest=~ copy=yes + when: result.stderr is defined and result.stderr != '' #有错误信息,说明docker服务不存在 + +- name: docker service not exist; install docker + shell: rpm -ivh ~/docker-rpm/*.rpm --force --nodeps && systemctl start docker && systemctl enable docker + when: result.stderr is defined and result.stderr != '' #有错误信息,说明docker服务不存在 + +- name: copy docker-compose.yml + template: + src: 'daemon.json.j2' + dest: '/etc/docker/daemon.json' + force: true + notify: #上一个任务的运行状态为changed后触发，否则不触发 + - restart docker + when: result.stderr is defined and result.stderr != '' + +#- name: sync daemon.json +# copy: src=file/daemon.json dest=/etc/docker/daemon.json force=true +# notify: #上一个任务的运行状态为changed后触发，否则不触发 +# - restart docker +# when: result.stderr is defined and result.stderr != '' #有错误信息,说明docker服务不存在 + +- name: check docker-compose;if docker compose does not exist, it will be print error;Don't Worry!!! + shell: docker-compose -v + ignore_errors: True + register: compose_result + +- name: docker-compose service not exist; copy docker-compose + copy: src=file/docker-compose-Linux-x86_64 dest=~/docker-compose-Linux-x86_64 force=true + when: compose_result.stderr is defined and compose_result.stderr != '' #有错误信息,说明docker-compose文件不存在或者没有执行权限 + +- name: docker-compose service not exist; install docker-compose + shell: cp ~/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose &&chmod +x /usr/local/bin/docker-compose + when: compose_result.stderr is defined and compose_result.stderr != '' #有错误信息,说明docker-compose文件不存在或者没有执行权限 diff --git a/parcels/roles/dependencies/install-docker/tasks/main.yml b/parcels/roles/dependencies/install-docker/tasks/main.yml new file mode 100644 index 0000000..47b4048 --- /dev/null +++ b/parcels/roles/dependencies/install-docker/tasks/main.yml @@ -0,0 +1 @@ +- include: install_docker_and_compose.yml diff --git a/parcels/roles/dependencies/install-docker/templates/daemon.json.j2 b/parcels/roles/dependencies/install-docker/templates/daemon.json.j2 new file mode 100644 index 0000000..7e455fb --- /dev/null +++ b/parcels/roles/dependencies/install-docker/templates/daemon.json.j2 @@ -0,0 +1,10 @@ +{ + "insecure-registries": [ + "10.16.43.130:5080" + ], + "registry-mirrors":[ + "https://d8b3zdiw.mirror.aliyuncs.com" + ], + "data-root": "{{ install_path }}/docker" +} + diff --git a/parcels/roles/dependencies/install-jdk/files/install_jdk.sh b/parcels/roles/dependencies/install-jdk/files/install_jdk.sh new file mode 100755 index 0000000..74d38ec --- /dev/null +++ b/parcels/roles/dependencies/install-jdk/files/install_jdk.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +#脚本与安装包需放在同一目录下 + +JDK_PATH=/usr/lib/jvm +JDK_HOME=$JDK_PATH/jdk1.8.0_73 + +echo "rpm -e java about openjdk....." + rpm -qa | grep openjdk | xargs rpm -e --nodeps + mkdir -p $JDK_PATH + +if ! grep "/usr/lib/jvm/jdk1.8.0_73" /etc/profile +then + echo "tar jdk-8u73-linux-x64.tar.gz to /usr/lib/jvm" + tar -xvzf jdk-8u73-linux-x64.tar.gz -C $JDK_PATH + echo "#set java enviroment to /etc/profile" + echo "#set java enviroment" >> /etc/profile + echo "export JAVA_HOME=$JDK_HOME" >> /etc/profile + echo "export JRE_HOME=\$JAVA_HOME/jre" >> /etc/profile + echo "export CLASSPATH=.:\$JAVA_HOME/lib:\$JRE_HOME/lib" >> /etc/profile + echo "export PATH=\$PATH:\$JAVA_HOME/bin" >> /etc/profile +fi + + source /etc/profile +clear +echo "jdk is installed !" diff --git a/parcels/roles/dependencies/install-jdk/files/jq-linux64 b/parcels/roles/dependencies/install-jdk/files/jq-linux64 Binary files differnew file mode 100755 index 0000000..939227e --- /dev/null +++ b/parcels/roles/dependencies/install-jdk/files/jq-linux64 diff --git a/parcels/roles/dependencies/install-jdk/tasks/install_jdk.yml b/parcels/roles/dependencies/install-jdk/tasks/install_jdk.yml new file mode 100644 index 0000000..f8e7c7e --- /dev/null +++ b/parcels/roles/dependencies/install-jdk/tasks/install_jdk.yml @@ -0,0 +1,44 @@ +- name: check jdk 1.8_73,if there are errors, don't worry!!! + shell: source /etc/profile && java -version 2>&1 | grep 1.8.0_73 | wc -l + ignore_errors: true + register: jdk_out #定义变量存储返回的结果 + +- name: create jdk directory + file: + path: '{{ jdk_home_path }}' + state: directory + when: jdk_out.stdout != '2' #等于2的时候说明jdk是安装好的 + +- name: copy jdk-8u73-linux-x64.tar.gz + copy: + src: '{{ package_path }}/jdk-8u73-linux-x64.tar.gz' + dest: '{{ jdk_home_path }}' + force: true + backup: yes + when: jdk_out.stdout != '2' + +- name: copy install_jdk.sh + copy: + src: '../files/install_jdk.sh' + dest: '{{ jdk_home_path }}' + force: true + mode: 0755 + backup: yes + when: jdk_out.stdout != '2' + + +#- name: copy file to {{ soft_home_path }} +# copy: src=file/ dest={{ soft_home_path }} force=true +# when: jdk_out.stdout != '2' #等于2的时候说明jdk是安装好的 + +- name: install the jdk,please wait for a moment + shell: cd {{ jdk_home_path }} && chmod +x install_jdk.sh && ./install_jdk.sh + when: jdk_out.stdout != '2' #等于2的时候说明jdk是安装好的 + +- name: install jq command + copy: + src: ../files/jq-linux64 + dest: '/usr/local/bin/jq' + force: true + mode: 0755 + backup: yes diff --git a/parcels/roles/dependencies/install-jdk/tasks/main.yml b/parcels/roles/dependencies/install-jdk/tasks/main.yml new file mode 100644 index 0000000..f5bdd9f --- /dev/null +++ b/parcels/roles/dependencies/install-jdk/tasks/main.yml @@ -0,0 +1 @@ +- include: install_jdk.yml diff --git a/parcels/roles/dependencies/install-jdk/vars/main.yml b/parcels/roles/dependencies/install-jdk/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/dependencies/install-jdk/vars/main.yml diff --git a/parcels/roles/dependencies/install-pip/file/PyMySQL-0.9.3-py2.py3-none-any.whl b/parcels/roles/dependencies/install-pip/file/PyMySQL-0.9.3-py2.py3-none-any.whl Binary files differnew file mode 100644 index 0000000..2c8a2f1 --- /dev/null +++ b/parcels/roles/dependencies/install-pip/file/PyMySQL-0.9.3-py2.py3-none-any.whl diff --git a/parcels/roles/dependencies/install-pip/file/docker-compose.zip b/parcels/roles/dependencies/install-pip/file/docker-compose.zip Binary files differnew file mode 100644 index 0000000..6b8a6a9 --- /dev/null +++ b/parcels/roles/dependencies/install-pip/file/docker-compose.zip diff --git a/parcels/roles/dependencies/install-pip/file/docker.zip b/parcels/roles/dependencies/install-pip/file/docker.zip Binary files differnew file mode 100644 index 0000000..375b8f4 --- /dev/null +++ b/parcels/roles/dependencies/install-pip/file/docker.zip diff --git a/parcels/roles/dependencies/install-pip/file/pip-19.2.2.tar.gz b/parcels/roles/dependencies/install-pip/file/pip-19.2.2.tar.gz Binary files differnew file mode 100644 index 0000000..2c9e034 --- /dev/null +++ b/parcels/roles/dependencies/install-pip/file/pip-19.2.2.tar.gz diff --git a/parcels/roles/dependencies/install-pip/file/pip.zip b/parcels/roles/dependencies/install-pip/file/pip.zip Binary files differnew file mode 100644 index 0000000..27e6848 --- /dev/null +++ b/parcels/roles/dependencies/install-pip/file/pip.zip diff --git a/parcels/roles/dependencies/install-pip/file/setuptools-41.1.0.post1.tar.gz b/parcels/roles/dependencies/install-pip/file/setuptools-41.1.0.post1.tar.gz Binary files differnew file mode 100644 index 0000000..6787726 --- /dev/null +++ b/parcels/roles/dependencies/install-pip/file/setuptools-41.1.0.post1.tar.gz diff --git a/parcels/roles/dependencies/install-pip/file/setuptools.zip b/parcels/roles/dependencies/install-pip/file/setuptools.zip Binary files differnew file mode 100644 index 0000000..5aaccc1 --- /dev/null +++ b/parcels/roles/dependencies/install-pip/file/setuptools.zip diff --git a/parcels/roles/dependencies/install-pip/tasks/install-pip-docker-compose.yml b/parcels/roles/dependencies/install-pip/tasks/install-pip-docker-compose.yml new file mode 100644 index 0000000..883b2ed --- /dev/null +++ b/parcels/roles/dependencies/install-pip/tasks/install-pip-docker-compose.yml @@ -0,0 +1,55 @@ +- name: copy gcc-python-devel.zip + unarchive: src={{ package_path }}/gcc-python-devel.zip dest=~ copy=yes + register: rpm_info + +#- debug: +# msg: "{{ rpm_info.changed }}" + +- name: rpm install gcc-python-devel + shell: rpm -ivh ~/gcc-python-devel/*.rpm --force --nodeps + when: rpm_info.changed + +- name: copy setuptools-41.1.0.post1.tar.gz + unarchive: src=file/setuptools-41.1.0.post1.tar.gz dest=~ copy=yes + when: rpm_info.changed + +- name: python install setuptools + shell: cd ~/setuptools-41.1.0.post1 && python setup.py install + when: rpm_info.changed + +- name: copy pip-19.2.2.tar.gz + unarchive: src=file/pip-19.2.2.tar.gz dest=~ copy=yes + when: rpm_info.changed + +- name: python install pip + shell: cd ~/pip-19.2.2 && python setup.py install + when: rpm_info.changed + +- name: copy docker.zip + unarchive: src=file/docker.zip dest=~ copy=yes + when: rpm_info.changed + +- name: pip install docker + shell: cd ~/docker && pip install --ignore-installed * + when: rpm_info.changed + +- name: copy docker-compose + unarchive: src=file/docker-compose.zip dest=~ copy=yes + when: rpm_info.changed + +- name: pip install docker-compose + shell: cd ~/docker-compose && pip install --ignore-installed * + when: rpm_info.changed +# ignore_errors: True +# register: command_result +# failed_when: "'FAILED' in command_result.stderr" + +- name: copy PyMySQL-0.9.3-py2.py3-none-any.whl to ~ + copy: + src: file/PyMySQL-0.9.3-py2.py3-none-any.whl + dest: '~' + force: true + backup: yes + +- name: pip install PyMysql + shell: cd ~ && pip install PyMySQL-0.9.3-py2.py3-none-any.whl diff --git a/parcels/roles/dependencies/install-pip/tasks/main.yml b/parcels/roles/dependencies/install-pip/tasks/main.yml new file mode 100644 index 0000000..8032e76 --- /dev/null +++ b/parcels/roles/dependencies/install-pip/tasks/main.yml @@ -0,0 +1 @@ +- include: install-pip-docker-compose.yml diff --git a/parcels/roles/init/clickhouse/files/create_ck_table.sql b/parcels/roles/init/clickhouse/files/create_ck_table.sql new file mode 100644 index 0000000..43c1c39 --- /dev/null +++ b/parcels/roles/init/clickhouse/files/create_ck_table.sql @@ -0,0 +1,4202 @@ +create database IF NOT EXISTS tsg_galaxy_v3 on cluster ck_cluster; +create database IF NOT EXISTS tsg_galaxy_v3 on cluster ck_query; + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record_log_local on cluster ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + gtp_version String, + gtp_apn String, + gtp_imei String, + gtp_imsi String, + gtp_phone_number String, + gtp_uplink_teid Int64, + gtp_downlink_teid Int64, + gtp_msg_type String, + gtp_end_user_ipv4 String, + gtp_end_user_ipv6 String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_log_id,common_data_center,common_recv_time); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record_log on cluster ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + gtp_version String, + gtp_apn String, + gtp_imei String, + gtp_imsi String, + gtp_phone_number String, + gtp_uplink_teid Int64, + gtp_downlink_teid Int64, + gtp_msg_type String, + gtp_end_user_ipv4 String, + gtp_end_user_ipv6 String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,gtpc_record_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record_log on cluster ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + gtp_version String, + gtp_apn String, + gtp_imei String, + gtp_imsi String, + gtp_phone_number String, + gtp_uplink_teid Int64, + gtp_downlink_teid Int64, + gtp_msg_type String, + gtp_end_user_ipv4 String, + gtp_end_user_ipv6 String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,gtpc_record_log_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.live_session_record_log_local on cluster ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_log_id,common_data_center,common_recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_log_local on cluster ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_stream_trace_id,common_data_center,common_recv_time); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.live_session_record_log on cluster ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,live_session_record_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.live_session_record_log on cluster ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,live_session_record_log_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_log on cluster ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_log on cluster ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_log_local,rand()); + + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_log_local on cluster ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +)ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_log_id,common_data_center,common_recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_log on cluster ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_log on cluster ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_log_local on cluster ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + nic_name String, + origin_source_mac String, + origin_dest_mac String, + packet_url String, + pcap_storage_task_id Int64, + pcap_storage_duration Int64 +) +ENGINE =MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_log_id,common_policy_id,common_recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_log_local on cluster ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 + +) +ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_log_id,common_policy_id,common_recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_log_local on cluster ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + doh_url String, + doh_host String, + doh_request_line String, + doh_response_line String, + doh_cookie String, + doh_referer String, + doh_user_agent String, + doh_content_length String, + doh_content_type String, + doh_set_cookie String, + doh_version String, + doh_message_id Int64, + doh_qr Int64, + doh_opcode Int64, + doh_aa Int64, + doh_tc Int64, + doh_rd Int64, + doh_ra Int64, + doh_rcode Int64, + doh_qdcount Int64, + doh_ancount Int64, + doh_nscount Int64, + doh_arcount Int64, + doh_qname String, + doh_qtype Int64, + doh_qclass Int64, + doh_cname String, + doh_sub Int64, + doh_rr String +) +ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_log_id,common_policy_id,common_recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_local on cluster ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_log_id,common_data_center,common_recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.radius_record_log_local on cluster ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + radius_packet_type Int64, + radius_nas_ip String, + radius_framed_ip String, + radius_account String, + radius_session_timeout Int64, + radius_idle_timeout Int64, + radius_acct_status_type Int64, + radius_acct_terminate_cause Int64, + radius_event_timestamp Int64, + radius_nas_port Int64, + radius_service_type Int64, + radius_framed_protocol Int64, + radius_callback_number String, + radius_callback_id String, + radius_termination_action Nullable(Int64), + radius_called_station_id String, + radius_calling_station_id String, + radius_acct_delay_time Int64, + radius_acct_session_id String, + radius_acct_multi_session_id String, + radius_acct_input_octets Int64, + radius_acct_output_octets Int64, + radius_acct_input_packets Int64, + radius_acct_output_packets Int64, + radius_acct_session_time Int64, + radius_acct_link_count Int64, + radius_acct_interim_interval Int64 +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_log_id,common_data_center,common_recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_client_ip_local on cluster ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_client_ip,common_server_ip,common_recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_server_ip_local on cluster ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_server_ip,common_client_ip,common_recv_time); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_subscriber_id_local on cluster ck_cluster ( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_subscriber_id,common_recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_http_domain_local on cluster ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (http_domain,common_recv_time); + + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip on cluster ck_cluster TO tsg_galaxy_v3.connection_record_log_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_policy_id, common_action, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni FROM tsg_galaxy_v3.connection_record_log_local; + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip on cluster ck_cluster TO tsg_galaxy_v3.connection_record_log_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_policy_id, common_action, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni FROM tsg_galaxy_v3.connection_record_log_local; + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_subscriber_id on cluster ck_cluster TO tsg_galaxy_v3.connection_record_log_common_subscriber_id_local AS SELECT common_log_id, common_recv_time, common_policy_id, common_action, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni FROM tsg_galaxy_v3.connection_record_log_local; + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain on cluster ck_cluster TO tsg_galaxy_v3.connection_record_log_http_domain_local AS SELECT common_log_id, common_recv_time, common_policy_id, common_action, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni FROM tsg_galaxy_v3.connection_record_log_local; + + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.radius_onff_log_local on cluster ck_cluster( + event_timestamp Int64, + account String, + framed_ip String, + acct_status_type Int64, + acct_session_id String, + acct_session_time Int64 +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(event_timestamp)) +ORDER BY (account,event_timestamp); + +create table IF NOT EXISTS tsg_galaxy_v3.radius_record_log on cluster ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + radius_packet_type Int64, + radius_nas_ip String, + radius_framed_ip String, + radius_account String, + radius_session_timeout Int64, + radius_idle_timeout Int64, + radius_acct_status_type Int64, + radius_acct_terminate_cause Int64, + radius_event_timestamp Int64, + radius_nas_port Int64, + radius_service_type Int64, + radius_framed_protocol Int64, + radius_callback_number String, + radius_callback_id String, + radius_termination_action Nullable(Int64), + radius_called_station_id String, + radius_calling_station_id String, + radius_acct_delay_time Int64, + radius_acct_session_id String, + radius_acct_multi_session_id String, + radius_acct_input_octets Int64, + radius_acct_output_octets Int64, + radius_acct_input_packets Int64, + radius_acct_output_packets Int64, + radius_acct_session_time Int64, + radius_acct_link_count Int64, + radius_acct_interim_interval Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_record_log_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.connection_record_log on cluster ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.proxy_event_log on cluster ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + doh_url String, + doh_host String, + doh_request_line String, + doh_response_line String, + doh_cookie String, + doh_referer String, + doh_user_agent String, + doh_content_length String, + doh_content_type String, + doh_set_cookie String, + doh_version String, + doh_message_id Int64, + doh_qr Int64, + doh_opcode Int64, + doh_aa Int64, + doh_tc Int64, + doh_rd Int64, + doh_ra Int64, + doh_rcode Int64, + doh_qdcount Int64, + doh_ancount Int64, + doh_nscount Int64, + doh_arcount Int64, + doh_qname String, + doh_qtype Int64, + doh_qclass Int64, + doh_cname String, + doh_sub Int64, + doh_rr String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_log_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.security_event_log on cluster ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_log_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_log on cluster ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + nic_name String, + origin_source_mac String, + origin_dest_mac String, + packet_url String, + pcap_storage_task_id Int64, + pcap_storage_duration Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,sys_packet_capture_log_local,rand()); + + +create table IF NOT EXISTS tsg_galaxy_v3.radius_onff_log on cluster ck_query( + event_timestamp Int64, + account String, + framed_ip String, + acct_status_type Int64, + acct_session_id String, + acct_session_time Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_onff_log_local,rand()); + + +create table IF NOT EXISTS tsg_galaxy_v3.radius_record_log on cluster ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + radius_packet_type Int64, + radius_nas_ip String, + radius_framed_ip String, + radius_account String, + radius_session_timeout Int64, + radius_idle_timeout Int64, + radius_acct_status_type Int64, + radius_acct_terminate_cause Int64, + radius_event_timestamp Int64, + radius_nas_port Int64, + radius_service_type Int64, + radius_framed_protocol Int64, + radius_callback_number String, + radius_callback_id String, + radius_termination_action Nullable(Int64), + radius_called_station_id String, + radius_calling_station_id String, + radius_acct_delay_time Int64, + radius_acct_session_id String, + radius_acct_multi_session_id String, + radius_acct_input_octets Int64, + radius_acct_output_octets Int64, + radius_acct_input_packets Int64, + radius_acct_output_packets Int64, + radius_acct_session_time Int64, + radius_acct_link_count Int64, + radius_acct_interim_interval Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_record_log_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.connection_record_log on cluster ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.proxy_event_log on cluster ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + doh_url String, + doh_host String, + doh_request_line String, + doh_response_line String, + doh_cookie String, + doh_referer String, + doh_user_agent String, + doh_content_length String, + doh_content_type String, + doh_set_cookie String, + doh_version String, + doh_message_id Int64, + doh_qr Int64, + doh_opcode Int64, + doh_aa Int64, + doh_tc Int64, + doh_rd Int64, + doh_ra Int64, + doh_rcode Int64, + doh_qdcount Int64, + doh_ancount Int64, + doh_nscount Int64, + doh_arcount Int64, + doh_qname String, + doh_qtype Int64, + doh_qclass Int64, + doh_cname String, + doh_sub Int64, + doh_rr String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_log_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.security_event_log on cluster ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_latency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_log_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_log on cluster ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_egress_link_id Nullable(Int64), + common_ingress_link_id Nullable(Int64), + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_userdefine_app_name String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + nic_name String, + origin_source_mac String, + origin_dest_mac String, + packet_url String, + pcap_storage_task_id Int64, + pcap_storage_duration Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,sys_packet_capture_log_local,rand()); + + +create table IF NOT EXISTS tsg_galaxy_v3.radius_onff_log on cluster ck_cluster( + event_timestamp Int64, + account String, + framed_ip String, + acct_status_type Int64, + acct_session_id String, + acct_session_time Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_onff_log_local,rand()); + + + + create table IF NOT EXISTS `system`.tables_cluster on cluster ck_query as `system`.tables ENGINE =Distributed(ck_all,`system`,tables,rand()); + create table IF NOT EXISTS `system`.disks_cluster on cluster ck_query as `system`.disks ENGINE =Distributed(ck_all,`system`,disks,rand()); + create table IF NOT EXISTS `system`.parts_cluster on cluster ck_query as `system`.parts ENGINE =Distributed(ck_all,`system`,parts,rand()); + create table IF NOT EXISTS `system`.query_log_cluster on cluster ck_query as `system`.query_log ENGINE =Distributed(ck_all,`system`,query_log,rand()); + CREATE TABLE IF NOT EXISTS `system`.columns_cluster ON cluster ck_query AS `system`.columns ENGINE=Distributed(ck_all,`system`,columns,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_client_ip on cluster ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_common_client_ip_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_server_ip on cluster ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_common_server_ip_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_subscriber_id on cluster ck_cluster ( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_common_subscriber_id_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_http_domain on cluster ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_http_domain_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_client_ip on cluster ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_common_client_ip_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_server_ip on cluster ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_common_server_ip_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_subscriber_id on cluster ck_query ( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_common_subscriber_id_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_http_domain on cluster ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_http_domain_local,rand()); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.active_defence_event_log_local on cluster ck_cluster( +common_log_id UInt64, +common_recv_time Int64, +common_entrance_id Int64, +common_device_id String, +common_egress_link_id Nullable(Int64), +common_ingress_link_id Nullable(Int64), +common_policy_id Int64, +common_user_region String, +ad_method String, +ad_protocol String, +common_address_type Int64, +ad_target_ip String, +ad_target_port String, +ad_cc_target_url String, +ad_target_ip_location String, +ad_target_ip_asn String, +ad_claimed_src_ip_profile_id Int64, +ad_reflector_profile_id Int64, +ad_sent_pkt_num Int64, +ad_sent_byte_num Int64, +ad_cc_initiate_connection_num Int64, +ad_cc_established_connection_num Int64, +ad_cc_rejected_connection_num Int64, +ad_generate_time Int64 +) +ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_log_id,common_policy_id,common_recv_time); + +create table IF NOT EXISTS tsg_galaxy_v3.active_defence_event_log on cluster ck_query( +common_log_id UInt64, +common_recv_time Int64, +common_entrance_id Int64, +common_device_id String, +common_egress_link_id Nullable(Int64), +common_ingress_link_id Nullable(Int64), +common_policy_id Int64, +common_user_region String, +ad_method String, +ad_protocol String, +common_address_type Int64, +ad_target_ip String, +ad_target_port String, +ad_cc_target_url String, +ad_target_ip_location String, +ad_target_ip_asn String, +ad_claimed_src_ip_profile_id Int64, +ad_reflector_profile_id Int64, +ad_sent_pkt_num Int64, +ad_sent_byte_num Int64, +ad_cc_initiate_connection_num Int64, +ad_cc_established_connection_num Int64, +ad_cc_rejected_connection_num Int64, +ad_generate_time Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,active_defence_event_log_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.active_defence_event_log on cluster ck_cluster( +common_log_id UInt64, +common_recv_time Int64, +common_entrance_id Int64, +common_device_id String, +common_egress_link_id Nullable(Int64), +common_ingress_link_id Nullable(Int64), +common_policy_id Int64, +common_user_region String, +ad_method String, +ad_protocol String, +common_address_type Int64, +ad_target_ip String, +ad_target_port String, +ad_cc_target_url String, +ad_target_ip_location String, +ad_target_ip_asn String, +ad_claimed_src_ip_profile_id Int64, +ad_reflector_profile_id Int64, +ad_sent_pkt_num Int64, +ad_sent_byte_num Int64, +ad_cc_initiate_connection_num Int64, +ad_cc_established_connection_num Int64, +ad_cc_rejected_connection_num Int64, +ad_generate_time Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,active_defence_event_log_local,rand()); + + + +CREATE DICTIONARY IF NOT EXISTS cdn on cluster ck_cluster ( +cdn_id UInt64, + domain String, + cname String) PRIMARY KEY cdn_id SOURCE(MYSQL(PORT 3306 USER 'root' PASSWORD 'bifang!@#' REPLICA (HOST '192.168.44.71' PRIORITY 1) DB 'tsg-bifang' TABLE 'tsg_cdn_domain_info')) LIFETIME(MIN 300 MAX 400) LAYOUT(FLAT()); + + create table IF NOT EXISTS tsg_galaxy_v3.cdn_dic on cluster ck_cluster (`cdn_id` UInt64, + `domain` String, + `cname` String) Engine = Dictionary(tsg_galaxy_v3.cdn); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_website_domain_info_local on cluster ck_cluster( + stat_time Int64, + policy_id Int64, + domain String, + ip_list AggregateFunction(groupUniqArray,String), + cdn_list AggregateFunction(groupUniqArray,String), + protocol_type_list AggregateFunction(groupUniqArray,String), + port_list AggregateFunction(groupUniqArray,Int64) +) ENGINE = AggregatingMergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (policy_id,domain,stat_time) SETTINGS index_granularity = 8192; + + + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_website_domain_info_local_view on cluster ck_cluster TO tsg_galaxy_v3.security_website_domain_info_local ( +stat_time UInt32, +policy_id Int64, +domain String, +ip_list AggregateFunction(groupUniqArray, + String), + cdn_list AggregateFunction(groupUniqArray, + String), + protocol_type_list AggregateFunction(groupUniqArray, + String), + port_list AggregateFunction(groupUniqArray, + Int64)) AS SELECT toUnixTimestamp(toStartOfDay(toDate(common_recv_time))) AS stat_time, + common_policy_id AS policy_id, + http_domain AS domain, + groupUniqArrayState(common_server_ip) AS ip_list, + groupUniqArrayState(cc.domain) AS cdn_list, + groupUniqArrayState(common_schema_type) AS protocol_type_list, + groupUniqArrayState(common_server_port) AS port_list FROM (SELECT common_recv_time, + common_policy_id, + http_domain, + common_server_ip, + arrayJoin(splitByChar(';', + replaceAll(ssl_san, + '*', + ''))) AS san, + common_schema_type, + common_server_port FROM tsg_galaxy_v3.security_event_log_local) AS sell INNER JOIN tsg_galaxy_v3.cdn_dic AS cc ON sell.san = cc.domain GROUP BY toStartOfDay(toDate(common_recv_time)), + common_policy_id, + http_domain; + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_ip_info_local on cluster ck_cluster ( + stat_time Int64, + policy_id Int64, + ip String, + domain_list AggregateFunction(groupUniqArray,String), + port_list AggregateFunction(groupUniqArray,Int64) +) ENGINE = AggregatingMergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (policy_id,ip,stat_time) SETTINGS index_granularity = 8192; + + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_ip_info_local_view on cluster ck_cluster TO tsg_galaxy_v3.security_ip_info_local AS +SELECT + toUnixTimestamp(toStartOfDay(toDate(common_recv_time))) as stat_time, + common_policy_id as policy_id, + common_server_ip as ip, + groupUniqArrayState(http_domain) as domain_list , + groupUniqArrayState(common_server_port) as port_list +FROM + tsg_galaxy_v3.security_event_log_local +group BY + toStartOfDay(toDate(common_recv_time)), + common_policy_id, + common_server_ip; + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_ip_info_local on cluster ck_cluster( + stat_time Int64, + policy_id Int64, + ip_list AggregateFunction(groupUniqArray,String) +) ENGINE = AggregatingMergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (policy_id,stat_time) SETTINGS index_granularity = 8192; + + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.proxy_ip_info_local_view on cluster ck_cluster TO tsg_galaxy_v3.proxy_ip_info_local ( + stat_time UInt32, + policy_id Int64, + ip_list AggregateFunction(groupUniqArray, + String)) AS SELECT toUnixTimestamp(toStartOfMinute(toDateTime(common_recv_time))) AS stat_time, + common_policy_id AS policy_id, + groupUniqArrayState(common_client_ip) AS ip_list FROM tsg_galaxy_v3.proxy_event_log_local GROUP BY toStartOfMinute(toDateTime(common_recv_time)), + common_policy_id; + +create table IF NOT EXISTS tsg_galaxy_v3.security_website_domain_info on cluster ck_query +( + stat_time Int64, + policy_id Int64, + domain String, + ip_list AggregateFunction(groupUniqArray,String), + cdn_list AggregateFunction(groupUniqArray,String), + protocol_type_list AggregateFunction(groupUniqArray,String), + port_list AggregateFunction(groupUniqArray,Int64) +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_website_domain_info_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.security_ip_info on cluster ck_query ( +stat_time Int64, + policy_id Int64, + ip String, + domain_list AggregateFunction(groupUniqArray,String), + port_list AggregateFunction(groupUniqArray,Int64) +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_ip_info_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.proxy_ip_info on cluster ck_query ( +stat_time Int64, + policy_id Int64, + ip_list AggregateFunction(groupUniqArray,String) +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_ip_info_local,rand()); + + diff --git a/parcels/roles/init/clickhouse/files/create_ck_table.sql_21.06 b/parcels/roles/init/clickhouse/files/create_ck_table.sql_21.06 new file mode 100644 index 0000000..2d4f882 --- /dev/null +++ b/parcels/roles/init/clickhouse/files/create_ck_table.sql_21.06 @@ -0,0 +1,4022 @@ +create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_cluster; +create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_query; + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record_log_local ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + gtp_version String, + gtp_apn String, + gtp_imei String, + gtp_imsi String, + gtp_phone_number String, + gtp_uplink_teid Int64, + gtp_downlink_teid Int64, + gtp_msg_type String, + gtp_end_user_ipv4 String, + gtp_end_user_ipv6 String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_log_id,common_data_center,common_recv_time); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record_log ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + gtp_version String, + gtp_apn String, + gtp_imei String, + gtp_imsi String, + gtp_phone_number String, + gtp_uplink_teid Int64, + gtp_downlink_teid Int64, + gtp_msg_type String, + gtp_end_user_ipv4 String, + gtp_end_user_ipv6 String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,gtpc_record_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.gtpc_record_log ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + gtp_version String, + gtp_apn String, + gtp_imei String, + gtp_imsi String, + gtp_phone_number String, + gtp_uplink_teid Int64, + gtp_downlink_teid Int64, + gtp_msg_type String, + gtp_end_user_ipv4 String, + gtp_end_user_ipv6 String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,gtpc_record_log_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.live_session_record_log_local ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_lantency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_log_id,common_data_center,common_recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_log_local ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_lantency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_stream_trace_id,common_data_center,common_recv_time); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.live_session_record_log ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_lantency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,live_session_record_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.live_session_record_log ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_lantency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,live_session_record_log_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_log ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_lantency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_log ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_lantency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_log_local,rand()); + + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_log_local ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +)ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_log_id,common_data_center,common_recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_log ON CLUSTER ck_query( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_log ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_log_local ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + nic_name String, + origin_source_mac String, + origin_dest_mac String, + packet_url String, + pcap_storage_task_id Int64, + pcap_storage_duration Int64 +) +ENGINE =MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_log_id,common_policy_id,common_recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_log_local ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_lantency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 + +) +ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_log_id,common_policy_id,common_recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_log_local ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_lantency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + doh_url String, + doh_host String, + doh_request_line String, + doh_response_line String, + doh_cookie String, + doh_referer String, + doh_user_agent String, + doh_content_length String, + doh_content_type String, + doh_set_cookie String, + doh_version String, + doh_message_id Int64, + doh_qr Int64, + doh_opcode Int64, + doh_aa Int64, + doh_tc Int64, + doh_rd Int64, + doh_ra Int64, + doh_rcode Int64, + doh_qdcount Int64, + doh_ancount Int64, + doh_nscount Int64, + doh_arcount Int64, + doh_qname String, + doh_qtype Int64, + doh_qclass Int64, + doh_cname String, + doh_sub Int64, + doh_rr String +) +ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_log_id,common_policy_id,common_recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_local ON CLUSTER ck_cluster( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_lantency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_log_id,common_data_center,common_recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.radius_record_log_local ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + radius_packet_type Int64, + radius_nas_ip String, + radius_framed_ip String, + radius_account String, + radius_session_timeout Int64, + radius_idle_timeout Int64, + radius_acct_status_type Int64, + radius_acct_terminate_cause Int64, + radius_event_timestamp Int64, + radius_nas_port Int64, + radius_service_type Int64, + radius_framed_protocol Int64, + radius_callback_number String, + radius_callback_id String, + radius_termination_action Nullable(Int64), + radius_called_station_id String, + radius_calling_station_id String, + radius_acct_delay_time Int64, + radius_acct_session_id String, + radius_acct_multi_session_id String, + radius_acct_input_octets Int64, + radius_acct_output_octets Int64, + radius_acct_input_packets Int64, + radius_acct_output_packets Int64, + radius_acct_session_time Int64, + radius_acct_link_count Int64, + radius_acct_interim_interval Int64 +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_log_id,common_data_center,common_recv_time); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_client_ip_local ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_client_ip,common_server_ip,common_recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_server_ip_local ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_server_ip,common_client_ip,common_recv_time); + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_subscriber_id_local ON CLUSTER ck_cluster ( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (common_subscriber_id,common_recv_time); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_http_domain_local ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) +ORDER BY (http_domain,common_recv_time); + + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.connection_record_log_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_policy_id, common_action, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni FROM tsg_galaxy_v3.connection_record_log_local; + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.connection_record_log_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_policy_id, common_action, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni FROM tsg_galaxy_v3.connection_record_log_local; + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_subscriber_id ON CLUSTER ck_cluster TO tsg_galaxy_v3.connection_record_log_common_subscriber_id_local AS SELECT common_log_id, common_recv_time, common_policy_id, common_action, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni FROM tsg_galaxy_v3.connection_record_log_local; + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.connection_record_log_http_domain_local AS SELECT common_log_id, common_recv_time, common_policy_id, common_action, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, http_domain, ssl_sni FROM tsg_galaxy_v3.connection_record_log_local; + + + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.radius_onff_log_local ON CLUSTER ck_cluster( + event_timestamp Int64, + account String, + framed_ip String, + acct_status_type Int64, + acct_session_id String, + acct_session_time Int64 +) +ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(event_timestamp)) +ORDER BY (account,event_timestamp); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.radius_record_log ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + radius_packet_type Int64, + radius_nas_ip String, + radius_framed_ip String, + radius_account String, + radius_session_timeout Int64, + radius_idle_timeout Int64, + radius_acct_status_type Int64, + radius_acct_terminate_cause Int64, + radius_event_timestamp Int64, + radius_nas_port Int64, + radius_service_type Int64, + radius_framed_protocol Int64, + radius_callback_number String, + radius_callback_id String, + radius_termination_action Nullable(Int64), + radius_called_station_id String, + radius_calling_station_id String, + radius_acct_delay_time Int64, + radius_acct_session_id String, + radius_acct_multi_session_id String, + radius_acct_input_octets Int64, + radius_acct_output_octets Int64, + radius_acct_input_packets Int64, + radius_acct_output_packets Int64, + radius_acct_session_time Int64, + radius_acct_link_count Int64, + radius_acct_interim_interval Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_record_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_lantency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_log ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_lantency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + doh_url String, + doh_host String, + doh_request_line String, + doh_response_line String, + doh_cookie String, + doh_referer String, + doh_user_agent String, + doh_content_length String, + doh_content_type String, + doh_set_cookie String, + doh_version String, + doh_message_id Int64, + doh_qr Int64, + doh_opcode Int64, + doh_aa Int64, + doh_tc Int64, + doh_rd Int64, + doh_ra Int64, + doh_rcode Int64, + doh_qdcount Int64, + doh_ancount Int64, + doh_nscount Int64, + doh_arcount Int64, + doh_qname String, + doh_qtype Int64, + doh_qclass Int64, + doh_cname String, + doh_sub Int64, + doh_rr String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_log ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_lantency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_log ON CLUSTER ck_query ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + nic_name String, + origin_source_mac String, + origin_dest_mac String, + packet_url String, + pcap_storage_task_id Int64, + pcap_storage_duration Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,sys_packet_capture_log_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.radius_onff_log ON CLUSTER ck_query( + event_timestamp Int64, + account String, + framed_ip String, + acct_status_type Int64, + acct_session_id String, + acct_session_time Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_onff_log_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.radius_record_log ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + radius_packet_type Int64, + radius_nas_ip String, + radius_framed_ip String, + radius_account String, + radius_session_timeout Int64, + radius_idle_timeout Int64, + radius_acct_status_type Int64, + radius_acct_terminate_cause Int64, + radius_event_timestamp Int64, + radius_nas_port Int64, + radius_service_type Int64, + radius_framed_protocol Int64, + radius_callback_number String, + radius_callback_id String, + radius_termination_action Nullable(Int64), + radius_called_station_id String, + radius_calling_station_id String, + radius_acct_delay_time Int64, + radius_acct_session_id String, + radius_acct_multi_session_id String, + radius_acct_input_octets Int64, + radius_acct_output_octets Int64, + radius_acct_input_packets Int64, + radius_acct_output_packets Int64, + radius_acct_session_time Int64, + radius_acct_link_count Int64, + radius_acct_interim_interval Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_record_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_lantency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_log ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_lantency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + doh_url String, + doh_host String, + doh_request_line String, + doh_response_line String, + doh_cookie String, + doh_referer String, + doh_user_agent String, + doh_content_length String, + doh_content_type String, + doh_set_cookie String, + doh_version String, + doh_message_id Int64, + doh_qr Int64, + doh_opcode Int64, + doh_aa Int64, + doh_tc Int64, + doh_rd Int64, + doh_ra Int64, + doh_rcode Int64, + doh_qdcount Int64, + doh_ancount Int64, + doh_nscount Int64, + doh_arcount Int64, + doh_qname String, + doh_qtype Int64, + doh_qclass Int64, + doh_cname String, + doh_sub Int64, + doh_rr String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_log ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + http_url String, + http_host String, + http_domain String, + http_request_line String, + http_response_line String, + http_request_header String, + http_response_header String, + http_request_content String, + http_response_content String, + http_request_body String, + http_response_body String, + http_request_body_key String, + http_response_body_key String, + http_proxy_flag Int64, + http_sequence Int64, + http_snapshot String, + http_cookie String, + http_referer String, + http_user_agent String, + http_content_length String, + http_content_type String, + http_set_cookie String, + http_version String, + http_response_lantency_ms Int64, + http_session_duration_ms Int64, + http_action_file_size Int64, + mail_protocol_type String, + mail_account String, + mail_to_cmd String, + mail_from_cmd String, + mail_from String, + mail_to String, + mail_cc String, + mail_bcc String, + mail_subject String, + mail_subject_charset String, + mail_content String, + mail_content_charset String, + mail_attachment_name String, + mail_attachment_name_charset String, + mail_attachment_content String, + mail_eml_file String, + mail_snapshot String, + dns_message_id Int64, + dns_qr Nullable(Int64), + dns_opcode Nullable(Int64), + dns_aa Int64, + dns_tc Int64, + dns_rd Int64, + dns_ra Int64, + dns_rcode Int64, + dns_qdcount Int64, + dns_ancount Int64, + dns_nscount Int64, + dns_arcount Int64, + dns_qname String, + dns_qtype Int64, + dns_qclass Int64, + dns_cname String, + dns_sub Int64, + dns_rr String, + ssl_version String, + ssl_sni String, + ssl_san String, + ssl_cn String, + ssl_pinningst Nullable(Int64), + ssl_intercept_state Nullable(Int64), + ssl_server_side_latency Int64, + ssl_client_side_latency Int64, + ssl_server_side_version String, + ssl_client_side_version String, + ssl_cert_verify Nullable(Int64), + ssl_error String, + ssl_con_latency_ms Int64, + ssl_ja3_fingerprint String, + ssl_ja3_hash String, + ssl_cert_issuer String, + ssl_cert_subject String, + quic_version String, + quic_sni String, + quic_user_agent String, + ftp_account String, + ftp_url String, + ftp_content String, + ftp_link_type String, + bgp_type Int64, + bgp_as_num String, + bgp_route String, + voip_calling_account String, + voip_called_account String, + voip_calling_number String, + voip_called_number String, + streaming_media_url String, + streaming_media_protocol String, + app_extra_info String, + sip_call_id String, + sip_originator_description String, + sip_responder_description String, + sip_user_agent String, + sip_server String, + sip_originator_sdp_connect_ip String, + sip_originator_sdp_media_port Int64, + sip_originator_sdp_media_type String, + sip_originator_sdp_content String, + sip_responder_sdp_connect_ip String, + sip_responder_sdp_media_port Int64, + sip_responder_sdp_media_type String, + sip_responder_sdp_content String, + sip_duration Int64, + sip_bye String, + rtp_payload_type_c2s Nullable(Int64), + rtp_payload_type_s2c Nullable(Int64), + rtp_pcap_path String, + rtp_originator_dir Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_log_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.sys_packet_capture_log ON CLUSTER ck_cluster ( + common_recv_time Int64, + common_log_id UInt64, + common_stream_trace_id UInt64, + common_direction Nullable(Int64), + common_stream_dir Int64, + common_start_time Int64, + common_end_time Int64, + common_con_duration_ms Int64, + common_establish_latency_ms Int64, + common_processing_time Int64, + common_entrance_id Int64, + common_device_id String, + common_link_id Int64, + common_isp String, + common_data_center String, + common_sled_ip String, + common_action Int64, + common_sub_action String, + common_policy_id Int64, + common_user_tags String, + common_user_region String, + common_client_ip String, + common_internal_ip String, + common_client_port Int64, + common_client_location String, + common_client_asn String, + common_subscriber_id String, + common_imei String, + common_imsi String, + common_phone_number String, + common_server_ip String, + common_external_ip String, + common_server_port Int64, + common_server_location String, + common_server_asn String, + common_app_id String, + common_app_label String, + common_app_surrogate_id String, + common_l7_protocol String, + common_protocol_label String, + common_service_category Array(Int64), + common_service Int64, + common_l4_protocol String, + common_sessions Int64, + common_c2s_pkt_num Int64, + common_s2c_pkt_num Int64, + common_c2s_pkt_diff Int64, + common_s2c_pkt_diff Int64, + common_c2s_byte_num Int64, + common_s2c_byte_num Int64, + common_c2s_byte_diff Int64, + common_s2c_byte_diff Int64, + common_c2s_ipfrag_num Int64, + common_s2c_ipfrag_num Int64, + common_c2s_tcp_lostlen Int64, + common_s2c_tcp_lostlen Int64, + common_c2s_tcp_unorder_num Int64, + common_s2c_tcp_unorder_num Int64, + common_c2s_pkt_retrans Int64, + common_s2c_pkt_retrans Int64, + common_c2s_byte_retrans Int64, + common_s2c_byte_retrans Int64, + common_first_ttl Int64, + common_tcp_client_isn Int64, + common_tcp_server_isn Int64, + common_address_type Int64, + common_schema_type String, + common_device_tag String, + common_encapsulation Int64, + common_tunnels String, + common_address_list String, + common_has_dup_traffic Int64, + common_stream_error String, + common_link_info_c2s String, + common_link_info_s2c String, + nic_name String, + origin_source_mac String, + origin_dest_mac String, + packet_url String, + pcap_storage_task_id Int64, + pcap_storage_duration Int64 +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,sys_packet_capture_log_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.radius_onff_log ON CLUSTER ck_cluster( + event_timestamp Int64, + account String, + framed_ip String, + acct_status_type Int64, + acct_session_id String, + acct_session_time Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,radius_onff_log_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_client_ip ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_common_client_ip_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_server_ip ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_common_server_ip_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_subscriber_id ON CLUSTER ck_cluster ( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_common_subscriber_id_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_http_domain ON CLUSTER ck_cluster( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_http_domain_local,rand()); + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_client_ip ON CLUSTER ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_common_client_ip_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_server_ip ON CLUSTER ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_common_server_ip_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_common_subscriber_id ON CLUSTER ck_query ( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_common_subscriber_id_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.connection_record_log_http_domain ON CLUSTER ck_query( + common_log_id UInt64, + common_recv_time Int64, + common_policy_id Int64, + common_action Int64, + common_server_ip String, + common_client_ip String, + common_sled_ip String, + common_entrance_id Int64, + common_subscriber_id String, + common_stream_trace_id UInt64, + http_domain String, + ssl_sni String +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,connection_record_log_http_domain_local,rand()); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.active_defence_event_log_local on cluster ck_cluster( +common_log_id UInt64, +common_recv_time Int64, +common_entrance_id Int64, +common_device_id String, +common_link_id Int64, +common_policy_id Int64, +common_user_region String, +ad_method String, +ad_protocol String, +common_address_type Int64, +ad_target_ip String, +ad_target_port String, +ad_cc_target_url String, +ad_target_ip_location String, +ad_target_ip_asn String, +ad_claimed_src_ip_profile_id Int64, +ad_reflector_profile_id Int64, +ad_sent_pkt_num Int64, +ad_sent_byte_num Int64, +ad_cc_initiate_connection_num Int64, +ad_cc_established_connection_num Int64, +ad_cc_rejected_connection_num Int64, +ad_generate_time Int64 +) +ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_log_id,common_policy_id,common_recv_time); + +create table IF NOT EXISTS tsg_galaxy_v3.active_defence_event_log on cluster ck_query( +common_log_id UInt64, +common_recv_time Int64, +common_entrance_id Int64, +common_device_id String, +common_link_id Int64, +common_policy_id Int64, +common_user_region String, +ad_method String, +ad_protocol String, +common_address_type Int64, +ad_target_ip String, +ad_target_port String, +ad_cc_target_url String, +ad_target_ip_location String, +ad_target_ip_asn String, +ad_claimed_src_ip_profile_id Int64, +ad_reflector_profile_id Int64, +ad_sent_pkt_num Int64, +ad_sent_byte_num Int64, +ad_cc_initiate_connection_num Int64, +ad_cc_established_connection_num Int64, +ad_cc_rejected_connection_num Int64, +ad_generate_time Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,active_defence_event_log_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.active_defence_event_log on cluster ck_cluster( +common_log_id UInt64, +common_recv_time Int64, +common_entrance_id Int64, +common_device_id String, +common_link_id Int64, +common_policy_id Int64, +common_user_region String, +ad_method String, +ad_protocol String, +common_address_type Int64, +ad_target_ip String, +ad_target_port String, +ad_cc_target_url String, +ad_target_ip_location String, +ad_target_ip_asn String, +ad_claimed_src_ip_profile_id Int64, +ad_reflector_profile_id Int64, +ad_sent_pkt_num Int64, +ad_sent_byte_num Int64, +ad_cc_initiate_connection_num Int64, +ad_cc_established_connection_num Int64, +ad_cc_rejected_connection_num Int64, +ad_generate_time Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,active_defence_event_log_local,rand()); + + + CREATE TABLE IF NOT EXISTS `system`.tables_cluster ON CLUSTER ck_query as `system`.tables ENGINE =Distributed(ck_all,`system`,tables,rand()); + CREATE TABLE IF NOT EXISTS `system`.disks_cluster ON CLUSTER ck_query as `system`.disks ENGINE =Distributed(ck_all,`system`,disks,rand()); + CREATE TABLE IF NOT EXISTS `system`.parts_cluster ON CLUSTER ck_query as `system`.parts ENGINE =Distributed(ck_all,`system`,parts,rand()); + CREATE TABLE IF NOT EXISTS `system`.query_log_cluster ON CLUSTER ck_query as `system`.query_log ENGINE =Distributed(ck_all,`system`,query_log,rand()); + CREATE TABLE IF NOT EXISTS `system`.columns_cluster ON cluster ck_query AS `system`.columns ENGINE=Distributed(ck_all,`system`,columns,rand()); + + diff --git a/parcels/roles/init/clickhouse/files/dll.sql b/parcels/roles/init/clickhouse/files/dll.sql new file mode 100644 index 0000000..db34b5a --- /dev/null +++ b/parcels/roles/init/clickhouse/files/dll.sql @@ -0,0 +1,200 @@ +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.active_defence_event_log_local on cluster ck_cluster( +common_log_id UInt64, +common_recv_time Int64, +common_entrance_id Int64, +common_device_id String, +common_link_id Int64, +common_policy_id Int64, +common_user_region String, +ad_method String, +ad_protocol String, +common_address_type Int64, +ad_target_ip String, +ad_target_port String, +ad_cc_target_url String, +ad_target_ip_location String, +ad_target_ip_asn String, +ad_claimed_src_ip_profile_id Int64, +ad_reflector_profile_id Int64, +ad_sent_pkt_num Int64, +ad_sent_byte_num Int64, +ad_cc_initiate_connection_num Int64, +ad_cc_established_connection_num Int64, +ad_cc_rejected_connection_num Int64, +ad_generate_time Int64 +) +ENGINE = MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time)) ORDER BY (common_log_id,common_policy_id,common_recv_time); + +create table IF NOT EXISTS tsg_galaxy_v3.active_defence_event_log on cluster ck_query( +common_log_id UInt64, +common_recv_time Int64, +common_entrance_id Int64, +common_device_id String, +common_link_id Int64, +common_policy_id Int64, +common_user_region String, +ad_method String, +ad_protocol String, +common_address_type Int64, +ad_target_ip String, +ad_target_port String, +ad_cc_target_url String, +ad_target_ip_location String, +ad_target_ip_asn String, +ad_claimed_src_ip_profile_id Int64, +ad_reflector_profile_id Int64, +ad_sent_pkt_num Int64, +ad_sent_byte_num Int64, +ad_cc_initiate_connection_num Int64, +ad_cc_established_connection_num Int64, +ad_cc_rejected_connection_num Int64, +ad_generate_time Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,active_defence_event_log_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.active_defence_event_log on cluster ck_cluster( +common_log_id UInt64, +common_recv_time Int64, +common_entrance_id Int64, +common_device_id String, +common_link_id Int64, +common_policy_id Int64, +common_user_region String, +ad_method String, +ad_protocol String, +common_address_type Int64, +ad_target_ip String, +ad_target_port String, +ad_cc_target_url String, +ad_target_ip_location String, +ad_target_ip_asn String, +ad_claimed_src_ip_profile_id Int64, +ad_reflector_profile_id Int64, +ad_sent_pkt_num Int64, +ad_sent_byte_num Int64, +ad_cc_initiate_connection_num Int64, +ad_cc_established_connection_num Int64, +ad_cc_rejected_connection_num Int64, +ad_generate_time Int64 +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,active_defence_event_log_local,rand()); + + + +CREATE DICTIONARY IF NOT EXISTS cdn on cluster ck_cluster ( +cdn_id UInt64, + domain String, + cname String) PRIMARY KEY cdn_id SOURCE(MYSQL(PORT 3306 USER 'root' PASSWORD 'bifang!@#' REPLICA (HOST '192.168.44.71' PRIORITY 1) DB 'tsg-bifang' TABLE 'tsg_cdn_domain_info')) LIFETIME(MIN 300 MAX 400) LAYOUT(FLAT()); + + create table IF NOT EXISTS tsg_galaxy_v3.cdn_dic on cluster ck_cluster (`cdn_id` UInt64, + `domain` String, + `cname` String) Engine = Dictionary(cdn); + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_website_domain_info_local on cluster ck_cluster( + stat_time Int64, + policy_id Int64, + domain String, + ip_list AggregateFunction(groupUniqArray,String), + cdn_list AggregateFunction(groupUniqArray,String), + protocol_type_list AggregateFunction(groupUniqArray,String), + port_list AggregateFunction(groupUniqArray,Int64) +) ENGINE = AggregatingMergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (policy_id,domain,stat_time) SETTINGS index_granularity = 8192; + + + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_website_domain_info_local_view on cluster ck_cluster TO tsg_galaxy_v3.security_website_domain_info_local ( +stat_time UInt32, +policy_id Int64, +domain String, +ip_list AggregateFunction(groupUniqArray, + String), + cdn_list AggregateFunction(groupUniqArray, + String), + protocol_type_list AggregateFunction(groupUniqArray, + String), + port_list AggregateFunction(groupUniqArray, + Int64)) AS SELECT toUnixTimestamp(toStartOfDay(toDate(common_recv_time))) AS stat_time, + common_policy_id AS policy_id, + http_domain AS domain, + groupUniqArrayState(common_server_ip) AS ip_list, + groupUniqArrayState(cc.domain) AS cdn_list, + groupUniqArrayState(common_schema_type) AS protocol_type_list, + groupUniqArrayState(common_server_port) AS port_list FROM (SELECT common_recv_time, + common_policy_id, + http_domain, + common_server_ip, + arrayJoin(splitByChar(';', + replaceAll(ssl_san, + '*', + ''))) AS san, + common_schema_type, + common_server_port FROM tsg_galaxy_v3.security_event_log_local) AS sell INNER JOIN tsg_galaxy_v3.cdn_dic AS cc ON sell.san = cc.domain GROUP BY toStartOfDay(toDate(common_recv_time)), + common_policy_id, + http_domain; + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_ip_info_local on cluster ck_cluster ( + stat_time Int64, + policy_id Int64, + ip String, + domain_list AggregateFunction(groupUniqArray,String), + port_list AggregateFunction(groupUniqArray,Int64) +) ENGINE = AggregatingMergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (policy_id,ip,stat_time) SETTINGS index_granularity = 8192; + + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_ip_info_local_view on cluster ck_cluster TO tsg_galaxy_v3.security_ip_info_local AS +SELECT + toUnixTimestamp(toStartOfDay(toDate(common_recv_time))) as stat_time, + common_policy_id as policy_id, + common_server_ip as ip, + groupUniqArrayState(http_domain) as domain_list , + groupUniqArrayState(common_server_port) as port_list +FROM + tsg_galaxy_v3.security_event_log_local +group BY + toStartOfDay(toDate(common_recv_time)), + common_policy_id, + common_server_ip; + + +CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_ip_info_local on cluster ck_cluster( + stat_time Int64, + policy_id Int64, + ip_list AggregateFunction(groupUniqArray,String) +) ENGINE = AggregatingMergeTree PARTITION BY toYYYYMMDD(toDate(stat_time)) ORDER BY (policy_id,stat_time) SETTINGS index_granularity = 8192; + + +CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.proxy_ip_info_local_view on cluster ck_cluster TO tsg_galaxy_v3.proxy_ip_info_local ( + stat_time UInt32, + policy_id Int64, + ip_list AggregateFunction(groupUniqArray, + String)) AS SELECT toUnixTimestamp(toStartOfMinute(toDateTime(common_recv_time))) AS stat_time, + common_policy_id AS policy_id, + groupUniqArrayState(common_client_ip) AS ip_list FROM tsg_galaxy_v3.proxy_event_log_local GROUP BY toStartOfMinute(toDateTime(common_recv_time)), + common_policy_id; + +create table IF NOT EXISTS tsg_galaxy_v3.security_website_domain_info on cluster ck_query +( + stat_time Int64, + policy_id Int64, + domain String, + ip_list AggregateFunction(groupUniqArray,String), + cdn_list AggregateFunction(groupUniqArray,String), + protocol_type_list AggregateFunction(groupUniqArray,String), + port_list AggregateFunction(groupUniqArray,Int64) +) +ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_website_domain_info_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.security_ip_info on cluster ck_query ( +stat_time Int64, + policy_id Int64, + ip String, + domain_list AggregateFunction(groupUniqArray,String), + port_list AggregateFunction(groupUniqArray,Int64) +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_ip_info_local,rand()); + +create table IF NOT EXISTS tsg_galaxy_v3.proxy_ip_info on cluster ck_query ( +stat_time Int64, + policy_id Int64, + ip_list AggregateFunction(groupUniqArray,String) +) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_ip_info_local,rand()); + diff --git a/parcels/roles/init/clickhouse/tasks/init_clickhouse.yml b/parcels/roles/init/clickhouse/tasks/init_clickhouse.yml new file mode 100644 index 0000000..b50e13e --- /dev/null +++ b/parcels/roles/init/clickhouse/tasks/init_clickhouse.yml @@ -0,0 +1,24 @@ +- name: query_ip to ansible variable + set_fact: query_ip="{{groups.clickhouse[0]}}" + +#复制脚本到目标服务器 +- name: copy create_ck_table.sql + copy: + src: '../files/create_ck_table.sql' + dest: '{{ install_path }}/' + force: true + backup: yes + run_once: true + delegate_to: "{{ query_ip }}" + +- name: create clickhouse tables + shell: clickhouse-client -h {{ inventory_hostname }} --port 9001 -m -u default --password {{ root_key }} --multiquery < {{ install_path }}/create_ck_table.sql + run_once: true + delegate_to: "{{ query_ip }}" + +- name: Ansible delete create_ck_table.sql + file: + path: "{{ install_path }}/create_ck_table.sql" + state: absent + run_once: true + delegate_to: "{{ query_ip }}" diff --git a/parcels/roles/init/clickhouse/tasks/main.yml b/parcels/roles/init/clickhouse/tasks/main.yml new file mode 100644 index 0000000..6528626 --- /dev/null +++ b/parcels/roles/init/clickhouse/tasks/main.yml @@ -0,0 +1 @@ +- include: init_clickhouse.yml diff --git a/parcels/roles/init/druid/files/druid_segments-tsg-21.05.sql b/parcels/roles/init/druid/files/druid_segments-tsg-21.05.sql new file mode 100644 index 0000000..8d005e0 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_segments-tsg-21.05.sql @@ -0,0 +1,62 @@ +/* + Navicat Premium Data Transfer + + Source Server : 192.168.44.226 + Source Server Type : MariaDB + Source Server Version : 100503 + Source Host : 192.168.44.226:3306 + Source Schema : druid_33 + + Target Server Type : MariaDB + Target Server Version : 100503 + File Encoding : 65001 + + Date: 02/04/2021 11:52:27 +*/ + +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for druid_segments +-- ---------------------------- +DROP TABLE IF EXISTS `druid_segments`; +CREATE TABLE `druid_segments` ( + `id` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + `dataSource` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `created_date` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + `start` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + `end` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + `partitioned` tinyint(1) NOT NULL, + `version` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + `used` tinyint(1) NOT NULL, + `payload` longblob NOT NULL, + PRIMARY KEY (`id`) USING BTREE, + INDEX `idx_druid_segments_used`(`used`) USING BTREE, + INDEX `idx_druid_segments_datasource_used_end`(`dataSource`, `used`, `end`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Records of druid_segments +-- ---------------------------- +INSERT INTO `druid_segments` VALUES ('client_internal_ip_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:39.273Z', 'client_internal_ip_hot_log', '2021-04-02T03:13:40.184Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:39.273Z', 1, 0x7B2264617461536F75726365223A22636C69656E745F696E7465726E616C5F69705F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A33392E3237335A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F636C69656E745F696E7465726E616C5F69705F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F33392E3237335A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22636F6D6D6F6E5F636C69656E745F69702C636F6D6D6F6E5F696E7465726E616C5F69702C636F6D6D6F6E5F6C345F70726F746F636F6C2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323630372C226964656E746966696572223A22636C69656E745F696E7465726E616C5F69705F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A33392E3237335A227D); +INSERT INTO `druid_segments` VALUES ('proxy_event_hits_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:41.641Z', 'proxy_event_hits_log', '2021-04-02T03:13:42.617Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:41.641Z', 1, 0x7B2264617461536F75726365223A2270726F78795F6576656E745F686974735F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A34312E3634315A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F70726F78795F6576656E745F686974735F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F34312E3634315A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22636F756E7472792C6973702C7375625F616374696F6E2C6C6F636174696F6E2C616374696F6E2C656E7472616E63655F69642C686974732C706F6C6963795F69642C6332735F627974655F6E756D2C7332635F627974655F6E756D222C226D657472696373223A2269705F6F626A656374222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A333239332C226964656E746966696572223A2270726F78795F6576656E745F686974735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A34312E3634315A227D); +INSERT INTO `druid_segments` VALUES ('proxy_event_urls_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:43.849Z', 'proxy_event_urls_hot_log', '2021-04-02T03:13:44.736Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:43.849Z', 1, 0x7B2264617461536F75726365223A2270726F78795F6576656E745F75726C735F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A34332E3834395A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F70726F78795F6576656E745F75726C735F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F34332E3834395A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A2275726C2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313031372C226964656E746966696572223A2270726F78795F6576656E745F75726C735F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A34332E3834395A227D); +INSERT INTO `druid_segments` VALUES ('security_event_hits_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:46.392Z', 'security_event_hits_log', '2021-04-02T03:13:47.468Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:46.392Z', 1, 0x7B2264617461536F75726365223A2273656375726974795F6576656E745F686974735F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A34362E3339325A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F73656375726974795F6576656E745F686974735F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F34362E3339325A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226973702C616374696F6E2C656E7472616E63655F69642C686974732C706F6C6963795F69642C6332735F627974655F6E756D2C7332635F627974655F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313939332C226964656E746966696572223A2273656375726974795F6576656E745F686974735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A34362E3339325A227D); +INSERT INTO `druid_segments` VALUES ('security_event_urls_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:48.458Z', 'security_event_urls_hot_log', '2021-04-02T03:13:49.325Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:48.458Z', 1, 0x7B2264617461536F75726365223A2273656375726974795F6576656E745F75726C735F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A34382E3435385A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F73656375726974795F6576656E745F75726C735F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F34382E3435385A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A2275726C2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313031372C226964656E746966696572223A2273656375726974795F6576656E745F75726C735F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A34382E3435385A227D); +INSERT INTO `druid_segments` VALUES ('server_external_ip_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:50.092Z', 'server_external_ip_hot_log', '2021-04-02T03:13:51.021Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:50.092Z', 1, 0x7B2264617461536F75726365223A227365727665725F65787465726E616C5F69705F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35302E3039325A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F7365727665725F65787465726E616C5F69705F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35302E3039325A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22636F6D6D6F6E5F7365727665725F69702C636F6D6D6F6E5F65787465726E616C5F69702C636F6D6D6F6E5F6C345F70726F746F636F6C2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323630372C226964656E746966696572223A227365727665725F65787465726E616C5F69705F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35302E3039325A227D); +INSERT INTO `druid_segments` VALUES ('sys_storage_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:52.202Z', 'sys_storage_log', '2021-04-02T03:13:53.057Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:52.202Z', 1, 0x7B2264617461536F75726365223A227379735F73746F726167655F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35322E3230325A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F7379735F73746F726167655F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35322E3230325A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226C6F675F747970652C646174615F63656E7465722C6167677265676174655F73697A652C6C6173745F73746F726167652C6D61785F73697A652C757365645F73697A65222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313938322C226964656E746966696572223A227379735F73746F726167655F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35322E3230325A227D); +INSERT INTO `druid_segments` VALUES ('top_client_ip_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:53.675Z', 'top_client_ip_log', '2021-04-02T03:13:54.681Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:53.675Z', 1, 0x7B2264617461536F75726365223A22746F705F636C69656E745F69705F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35332E3637355A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F636C69656E745F69705F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35332E3637355A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22736F757263652C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323137372C226964656E746966696572223A22746F705F636C69656E745F69705F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35332E3637355A227D); +INSERT INTO `druid_segments` VALUES ('top_external_host_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:55.845Z', 'top_external_host_log', '2021-04-02T03:13:57.329Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:55.845Z', 1, 0x7B2264617461536F75726365223A22746F705F65787465726E616C5F686F73745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35352E3834355A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F65787465726E616C5F686F73745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35352E3834355A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A2264657374696E6174696F6E2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323139322C226964656E746966696572223A22746F705F65787465726E616C5F686F73745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35352E3834355A227D); +INSERT INTO `druid_segments` VALUES ('top_internal_host_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:58.433Z', 'top_internal_host_log', '2021-04-02T03:13:59.446Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:58.433Z', 1, 0x7B2264617461536F75726365223A22746F705F696E7465726E616C5F686F73745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35382E3433335A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F696E7465726E616C5F686F73745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35382E3433335A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22736F757263652C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323137372C226964656E746966696572223A22746F705F696E7465726E616C5F686F73745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35382E3433335A227D); +INSERT INTO `druid_segments` VALUES ('top_server_ip_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:59.654Z', 'top_server_ip_log', '2021-04-02T03:14:00.955Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:59.654Z', 1, 0x7B2264617461536F75726365223A22746F705F7365727665725F69705F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35392E3635345A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F7365727665725F69705F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35392E3635345A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A2264657374696E6174696F6E2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323139322C226964656E746966696572223A22746F705F7365727665725F69705F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35392E3635345A227D); +INSERT INTO `druid_segments` VALUES ('top_urls_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:01.484Z', 'top_urls_log', '2021-04-02T03:14:02.269Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:01.484Z', 1, 0x7B2264617461536F75726365223A22746F705F75726C735F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A30312E3438345A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F75726C735F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F30312E3438345A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A2275726C2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313031372C226964656E746966696572223A22746F705F75726C735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A30312E3438345A227D); +INSERT INTO `druid_segments` VALUES ('top_user_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:04.063Z', 'top_user_log', '2021-04-02T03:14:04.955Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:04.063Z', 1, 0x7B2264617461536F75726365223A22746F705F757365725F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A30342E3036335A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F757365725F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F30342E3036335A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22737562736372696265725F69642C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323139382C226964656E746966696572223A22746F705F757365725F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A30342E3036335A227D); +INSERT INTO `druid_segments` VALUES ('top_website_domain_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:06.100Z', 'top_website_domain_log', '2021-04-02T03:14:07.150Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:06.100Z', 1, 0x7B2264617461536F75726365223A22746F705F776562736974655F646F6D61696E5F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A30362E3130305A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F776562736974655F646F6D61696E5F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F30362E3130305A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22646F6D61696E2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323137372C226964656E746966696572223A22746F705F776562736974655F646F6D61696E5F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A30362E3130305A227D); +INSERT INTO `druid_segments` VALUES ('traffic_metrics_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:08.462Z', 'traffic_metrics_log', '2021-04-02T03:14:09.342Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:08.462Z', 1, 0x7B2264617461536F75726365223A22747261666669635F6D6574726963735F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A30382E3436325A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F6D6574726963735F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F30382E3436325A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226465766963655F69642C616C6C6F775F636F6E6E5F6E756D2C616C6C6F775F696E5F62797465732C616C6C6F775F696E5F7061636B6574732C616C6C6F775F6F75745F62797465732C616C6C6F775F6F75745F7061636B6574732C636C6F73655F636F6E6E5F6E756D2C64656661756C745F636F6E6E5F6E756D2C64656661756C745F696E5F62797465732C64656661756C745F696E5F7061636B6574732C64656661756C745F6F75745F62797465732C64656661756C745F6F75745F7061636B6574732C64656E795F636F6E6E5F6E756D2C64656E795F696E5F62797465732C64656E795F696E5F7061636B6574732C64656E795F6F75745F62797465732C64656E795F6F75745F7061636B6574732C65737461626C69736865645F636F6E6E5F6E756D2C696E746572636570745F636F6E6E5F6E756D2C696E746572636570745F696E5F62797465732C696E746572636570745F696E5F7061636B6574732C696E746572636570745F6F75745F62797465732C696E746572636570745F6F75745F7061636B6574732C6D6F6E69746F725F636F6E6E5F6E756D2C6D6F6E69746F725F696E5F62797465732C6D6F6E69746F725F696E5F7061636B6574732C6D6F6E69746F725F6F75745F62797465732C6D6F6E69746F725F6F75745F7061636B6574732C6E65775F636F6E6E5F6E756D2C746F74616C5F696E5F62797465732C746F74616C5F696E5F7061636B6574732C746F74616C5F6F75745F62797465732C656E7472616E63655F69642C746F74616C5F6F75745F7061636B6574732C70696E6E696E675F6E756D2C6E6F745F70696E6E696E675F6E756D2C6D617962655F70696E6E696E675F6E756D2C616C6572745F62797465732C696E735F62797465732C68696A6B5F62797465732C61645F7265666C656374696F6E5F62797465732C61645F666C6F6F645F62797465732C61645F63635F62797465732C696E7463705F6D6F6E5F6E756D2C696E7463705F616C6C6F775F6E756D2C696E7463705F64656E795F6E756D2C696E7463705F72646972745F6E756D2C696E7463705F7265706C5F6E756D2C696E7463705F68696A6B5F6E756D2C696E7463705F696E735F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A31313430362C226964656E746966696572223A22747261666669635F6D6574726963735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A30382E3436325A227D); +INSERT INTO `druid_segments` VALUES ('traffic_protocol_stat_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:09.706Z', 'traffic_protocol_stat_log', '2021-04-02T03:14:10.583Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:09.706Z', 1, 0x7B2264617461536F75726365223A22747261666669635F70726F746F636F6C5F737461745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A30392E3730365A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F70726F746F636F6C5F737461745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F30392E3730365A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226973702C70726F746F636F6C5F69642C646174615F63656E7465722C6332735F706B745F6E756D2C656E7472616E63655F69642C73657373696F6E732C7332635F706B745F6E756D2C6332735F627974655F6E756D2C7332635F627974655F6E756D2C6332735F6970667261675F6E756D2C7332635F6970667261675F6E756D2C6332735F7463705F6C6F73746C656E2C7332635F7463705F6C6F73746C656E2C6332735F7463705F756E6F726465725F6E756D2C7332635F7463705F756E6F726465725F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A343032372C226964656E746966696572223A22747261666669635F70726F746F636F6C5F737461745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A30392E3730365A227D); +INSERT INTO `druid_segments` VALUES ('traffic_summary_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:12.193Z', 'traffic_summary_log', '2021-04-02T03:14:13.233Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:12.193Z', 1, 0x7B2264617461536F75726365223A22747261666669635F73756D6D6172795F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A31322E3139335A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F73756D6D6172795F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F31322E3139335A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226973702C736368656D615F747970652C646174615F63656E7465722C6332735F706B745F6E756D2C656E7472616E63655F69642C73657373696F6E732C7332635F706B745F6E756D2C6332735F627974655F6E756D2C7332635F627974655F6E756D2C6F6E655F73696465645F636F6E6E656374696F6E732C756E63617465676F72697A65645F62797465732C667261676D656E746174696F6E5F7061636B6574732C73657175656E63655F6761705F6C6F73732C756E6F726465725F7061636B657473222C226D657472696373223A2269705F6F626A656374222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A343133342C226964656E746966696572223A22747261666669635F73756D6D6172795F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A31322E3139335A227D); +INSERT INTO `druid_segments` VALUES ('user_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:14.350Z', 'user_hot_log', '2021-04-02T03:14:15.137Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:14.350Z', 1, 0x7B2264617461536F75726365223A22757365725F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A31342E3335305A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F757365725F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F31342E3335305A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22636F6D6D6F6E5F737562736372696265725F69642C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313838332C226964656E746966696572223A22757365725F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A31342E3335305A227D); +INSERT INTO `druid_segments` VALUES ('website_domain_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:16.425Z', 'website_domain_hot_log', '2021-04-02T03:14:17.174Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:16.425Z', 1, 0x7B2264617461536F75726365223A22776562736974655F646F6D61696E5F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A31362E3432355A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F776562736974655F646F6D61696E5F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F31362E3432355A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22646F6D61696E2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313834312C226964656E746966696572223A22776562736974655F646F6D61696E5F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A31362E3432355A227D); + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/parcels/roles/init/druid/files/druid_segments-tsg3.0.sql b/parcels/roles/init/druid/files/druid_segments-tsg3.0.sql new file mode 100644 index 0000000..ceb9d8a --- /dev/null +++ b/parcels/roles/init/druid/files/druid_segments-tsg3.0.sql @@ -0,0 +1,63 @@ +/* + Navicat Premium Data Transfer + + Source Server : 192.168.44.226 + Source Server Type : MariaDB + Source Server Version : 100503 + Source Host : 192.168.44.226:3306 + Source Schema : druid_33 + + Target Server Type : MariaDB + Target Server Version : 100503 + File Encoding : 65001 + + Date: 02/04/2021 11:52:27 +*/ + +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for druid_segments +-- ---------------------------- +DROP TABLE IF EXISTS `druid_segments`; +CREATE TABLE `druid_segments` ( + `id` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + `dataSource` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + `created_date` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + `start` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + `end` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + `partitioned` tinyint(1) NOT NULL, + `version` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, + `used` tinyint(1) NOT NULL, + `payload` longblob NOT NULL, + PRIMARY KEY (`id`) USING BTREE, + INDEX `idx_druid_segments_used`(`used`) USING BTREE, + INDEX `idx_druid_segments_datasource_used_end`(`dataSource`, `used`, `end`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Records of druid_segments +-- ---------------------------- +INSERT INTO `druid_segments` VALUES ('client_internal_ip_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:39.273Z', 'client_internal_ip_hot_log', '2021-04-02T03:13:40.184Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:39.273Z', 1, 0x7B2264617461536F75726365223A22636C69656E745F696E7465726E616C5F69705F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A33392E3237335A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F636C69656E745F696E7465726E616C5F69705F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F33392E3237335A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22636F6D6D6F6E5F636C69656E745F69702C636F6D6D6F6E5F696E7465726E616C5F69702C636F6D6D6F6E5F6C345F70726F746F636F6C2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323630372C226964656E746966696572223A22636C69656E745F696E7465726E616C5F69705F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A33392E3237335A227D); +INSERT INTO `druid_segments` VALUES ('proxy_event_hits_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:41.641Z', 'proxy_event_hits_log', '2021-04-02T03:13:42.617Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:41.641Z', 1, 0x7B2264617461536F75726365223A2270726F78795F6576656E745F686974735F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A34312E3634315A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F70726F78795F6576656E745F686974735F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F34312E3634315A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22636F756E7472792C6973702C7375625F616374696F6E2C6C6F636174696F6E2C616374696F6E2C656E7472616E63655F69642C686974732C706F6C6963795F69642C6332735F627974655F6E756D2C7332635F627974655F6E756D222C226D657472696373223A2269705F6F626A656374222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A333239332C226964656E746966696572223A2270726F78795F6576656E745F686974735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A34312E3634315A227D); +INSERT INTO `druid_segments` VALUES ('proxy_event_urls_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:43.849Z', 'proxy_event_urls_hot_log', '2021-04-02T03:13:44.736Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:43.849Z', 1, 0x7B2264617461536F75726365223A2270726F78795F6576656E745F75726C735F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A34332E3834395A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F70726F78795F6576656E745F75726C735F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F34332E3834395A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A2275726C2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313031372C226964656E746966696572223A2270726F78795F6576656E745F75726C735F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A34332E3834395A227D); +INSERT INTO `druid_segments` VALUES ('security_event_hits_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:46.392Z', 'security_event_hits_log', '2021-04-02T03:13:47.468Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:46.392Z', 1, 0x7B2264617461536F75726365223A2273656375726974795F6576656E745F686974735F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A34362E3339325A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F73656375726974795F6576656E745F686974735F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F34362E3339325A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226973702C616374696F6E2C656E7472616E63655F69642C686974732C706F6C6963795F69642C6332735F627974655F6E756D2C7332635F627974655F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313939332C226964656E746966696572223A2273656375726974795F6576656E745F686974735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A34362E3339325A227D); +INSERT INTO `druid_segments` VALUES ('security_event_urls_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:48.458Z', 'security_event_urls_hot_log', '2021-04-02T03:13:49.325Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:48.458Z', 1, 0x7B2264617461536F75726365223A2273656375726974795F6576656E745F75726C735F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A34382E3435385A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F73656375726974795F6576656E745F75726C735F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F34382E3435385A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A2275726C2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313031372C226964656E746966696572223A2273656375726974795F6576656E745F75726C735F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A34382E3435385A227D); +INSERT INTO `druid_segments` VALUES ('server_external_ip_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:50.092Z', 'server_external_ip_hot_log', '2021-04-02T03:13:51.021Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:50.092Z', 1, 0x7B2264617461536F75726365223A227365727665725F65787465726E616C5F69705F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35302E3039325A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F7365727665725F65787465726E616C5F69705F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35302E3039325A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22636F6D6D6F6E5F7365727665725F69702C636F6D6D6F6E5F65787465726E616C5F69702C636F6D6D6F6E5F6C345F70726F746F636F6C2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323630372C226964656E746966696572223A227365727665725F65787465726E616C5F69705F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35302E3039325A227D); +INSERT INTO `druid_segments` VALUES ('sys_storage_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:52.202Z', 'sys_storage_log', '2021-04-02T03:13:53.057Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:52.202Z', 1, 0x7B2264617461536F75726365223A227379735F73746F726167655F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35322E3230325A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F7379735F73746F726167655F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35322E3230325A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226C6F675F747970652C646174615F63656E7465722C6167677265676174655F73697A652C6C6173745F73746F726167652C6D61785F73697A652C757365645F73697A65222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313938322C226964656E746966696572223A227379735F73746F726167655F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35322E3230325A227D); +INSERT INTO `druid_segments` VALUES ('top_client_ip_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:53.675Z', 'top_client_ip_log', '2021-04-02T03:13:54.681Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:53.675Z', 1, 0x7B2264617461536F75726365223A22746F705F636C69656E745F69705F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35332E3637355A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F636C69656E745F69705F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35332E3637355A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22736F757263652C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323137372C226964656E746966696572223A22746F705F636C69656E745F69705F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35332E3637355A227D); +INSERT INTO `druid_segments` VALUES ('top_external_host_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:55.845Z', 'top_external_host_log', '2021-04-02T03:13:57.329Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:55.845Z', 1, 0x7B2264617461536F75726365223A22746F705F65787465726E616C5F686F73745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35352E3834355A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F65787465726E616C5F686F73745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35352E3834355A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A2264657374696E6174696F6E2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323139322C226964656E746966696572223A22746F705F65787465726E616C5F686F73745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35352E3834355A227D); +INSERT INTO `druid_segments` VALUES ('top_internal_host_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:58.433Z', 'top_internal_host_log', '2021-04-02T03:13:59.446Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:58.433Z', 1, 0x7B2264617461536F75726365223A22746F705F696E7465726E616C5F686F73745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35382E3433335A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F696E7465726E616C5F686F73745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35382E3433335A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22736F757263652C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323137372C226964656E746966696572223A22746F705F696E7465726E616C5F686F73745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35382E3433335A227D); +INSERT INTO `druid_segments` VALUES ('top_server_ip_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:13:59.654Z', 'top_server_ip_log', '2021-04-02T03:14:00.955Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:13:59.654Z', 1, 0x7B2264617461536F75726365223A22746F705F7365727665725F69705F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31333A35392E3635345A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F7365727665725F69705F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31335F35392E3635345A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A2264657374696E6174696F6E2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323139322C226964656E746966696572223A22746F705F7365727665725F69705F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31333A35392E3635345A227D); +INSERT INTO `druid_segments` VALUES ('top_urls_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:01.484Z', 'top_urls_log', '2021-04-02T03:14:02.269Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:01.484Z', 1, 0x7B2264617461536F75726365223A22746F705F75726C735F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A30312E3438345A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F75726C735F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F30312E3438345A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A2275726C2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313031372C226964656E746966696572223A22746F705F75726C735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A30312E3438345A227D); +INSERT INTO `druid_segments` VALUES ('top_user_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:04.063Z', 'top_user_log', '2021-04-02T03:14:04.955Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:04.063Z', 1, 0x7B2264617461536F75726365223A22746F705F757365725F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A30342E3036335A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F757365725F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F30342E3036335A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22737562736372696265725F69642C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323139382C226964656E746966696572223A22746F705F757365725F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A30342E3036335A227D); +INSERT INTO `druid_segments` VALUES ('top_website_domain_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:06.100Z', 'top_website_domain_log', '2021-04-02T03:14:07.150Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:06.100Z', 1, 0x7B2264617461536F75726365223A22746F705F776562736974655F646F6D61696E5F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A30362E3130305A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F746F705F776562736974655F646F6D61696E5F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F30362E3130305A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22646F6D61696E2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C6F726465725F6279222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323137372C226964656E746966696572223A22746F705F776562736974655F646F6D61696E5F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A30362E3130305A227D); +INSERT INTO `druid_segments` VALUES ('traffic_metrics_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:08.462Z', 'traffic_metrics_log', '2021-04-02T03:14:09.342Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:08.462Z', 1, 0x7B2264617461536F75726365223A22747261666669635F6D6574726963735F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A30382E3436325A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F6D6574726963735F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F30382E3436325A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226465766963655F69642C616C6C6F775F636F6E6E5F6E756D2C616C6C6F775F696E5F62797465732C616C6C6F775F696E5F7061636B6574732C616C6C6F775F6F75745F62797465732C616C6C6F775F6F75745F7061636B6574732C636C6F73655F636F6E6E5F6E756D2C64656661756C745F636F6E6E5F6E756D2C64656661756C745F696E5F62797465732C64656661756C745F696E5F7061636B6574732C64656661756C745F6F75745F62797465732C64656661756C745F6F75745F7061636B6574732C64656E795F636F6E6E5F6E756D2C64656E795F696E5F62797465732C64656E795F696E5F7061636B6574732C64656E795F6F75745F62797465732C64656E795F6F75745F7061636B6574732C65737461626C69736865645F636F6E6E5F6E756D2C696E746572636570745F636F6E6E5F6E756D2C696E746572636570745F696E5F62797465732C696E746572636570745F696E5F7061636B6574732C696E746572636570745F6F75745F62797465732C696E746572636570745F6F75745F7061636B6574732C6D6F6E69746F725F636F6E6E5F6E756D2C6D6F6E69746F725F696E5F62797465732C6D6F6E69746F725F696E5F7061636B6574732C6D6F6E69746F725F6F75745F62797465732C6D6F6E69746F725F6F75745F7061636B6574732C6E65775F636F6E6E5F6E756D2C746F74616C5F696E5F62797465732C746F74616C5F696E5F7061636B6574732C746F74616C5F6F75745F62797465732C656E7472616E63655F69642C746F74616C5F6F75745F7061636B6574732C70696E6E696E675F6E756D2C6E6F745F70696E6E696E675F6E756D2C6D617962655F70696E6E696E675F6E756D2C616C6572745F62797465732C696E735F62797465732C68696A6B5F62797465732C61645F7265666C656374696F6E5F62797465732C61645F666C6F6F645F62797465732C61645F63635F62797465732C696E7463705F6D6F6E5F6E756D2C696E7463705F616C6C6F775F6E756D2C696E7463705F64656E795F6E756D2C696E7463705F72646972745F6E756D2C696E7463705F7265706C5F6E756D2C696E7463705F68696A6B5F6E756D2C696E7463705F696E735F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A31313430362C226964656E746966696572223A22747261666669635F6D6574726963735F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A30382E3436325A227D); +INSERT INTO `druid_segments` VALUES ('traffic_protocol_stat_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:09.706Z', 'traffic_protocol_stat_log', '2021-04-02T03:14:10.583Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:09.706Z', 1, 0x7B2264617461536F75726365223A22747261666669635F70726F746F636F6C5F737461745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A30392E3730365A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F70726F746F636F6C5F737461745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F30392E3730365A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226973702C70726F746F636F6C5F69642C646174615F63656E7465722C6332735F706B745F6E756D2C656E7472616E63655F69642C73657373696F6E732C7332635F706B745F6E756D2C6332735F627974655F6E756D2C7332635F627974655F6E756D2C6332735F6970667261675F6E756D2C7332635F6970667261675F6E756D2C6332735F7463705F6C6F73746C656E2C7332635F7463705F6C6F73746C656E2C6332735F7463705F756E6F726465725F6E756D2C7332635F7463705F756E6F726465725F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A343032372C226964656E746966696572223A22747261666669635F70726F746F636F6C5F737461745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A30392E3730365A227D); +INSERT INTO `druid_segments` VALUES ('traffic_summary_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:12.193Z', 'traffic_summary_log', '2021-04-02T03:14:13.233Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:12.193Z', 1, 0x7B2264617461536F75726365223A22747261666669635F73756D6D6172795F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A31322E3139335A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F73756D6D6172795F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F31322E3139335A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226973702C736368656D615F747970652C646174615F63656E7465722C6332735F706B745F6E756D2C656E7472616E63655F69642C73657373696F6E732C7332635F706B745F6E756D2C6332735F627974655F6E756D2C7332635F627974655F6E756D2C6F6E655F73696465645F636F6E6E656374696F6E732C756E63617465676F72697A65645F62797465732C667261676D656E746174696F6E5F7061636B6574732C73657175656E63655F6761705F6C6F73732C756E6F726465725F7061636B657473222C226D657472696373223A2269705F6F626A656374222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A343133342C226964656E746966696572223A22747261666669635F73756D6D6172795F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A31322E3139335A227D); +INSERT INTO `druid_segments` VALUES ('user_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:14.350Z', 'user_hot_log', '2021-04-02T03:14:15.137Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:14.350Z', 1, 0x7B2264617461536F75726365223A22757365725F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A31342E3335305A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F757365725F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F31342E3335305A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22636F6D6D6F6E5F737562736372696265725F69642C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313838332C226964656E746966696572223A22757365725F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A31342E3335305A227D); +INSERT INTO `druid_segments` VALUES ('website_domain_hot_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-04-02T03:14:16.425Z', 'website_domain_hot_log', '2021-04-02T03:14:17.174Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-04-02T03:14:16.425Z', 1, 0x7B2264617461536F75726365223A22776562736974655F646F6D61696E5F686F745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30342D30325430333A31343A31362E3432355A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F776562736974655F646F6D61696E5F686F745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30342D30325430335F31345F31362E3432355A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A22646F6D61696E2C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A313834312C226964656E746966696572223A22776562736974655F646F6D61696E5F686F745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30342D30325430333A31343A31362E3432355A227D); +INSERT INTO `druid_segments` VALUES ('traffic_app_stat_log_3000-01-02T00:00:00.000Z_3000-01-03T00:00:00.000Z_2021-07-02T02:43:09.615Z', 'traffic_app_stat_log', '2021-07-02T02:43:10.572Z', '3000-01-02T00:00:00.000Z', '3000-01-03T00:00:00.000Z', 1, '2021-07-02T02:43:09.615Z', 1, 0x7B2264617461536F75726365223A22747261666669635F6170705F737461745F6C6F67222C22696E74657276616C223A22333030302D30312D30325430303A30303A30302E3030305A2F333030302D30312D30335430303A30303A30302E3030305A222C2276657273696F6E223A22323032312D30372D30325430323A34333A30392E3631355A222C226C6F616453706563223A7B2274797065223A2268646673222C2270617468223A22686466733A2F2F6E73312F64727569642F7365676D656E74732F747261666669635F6170705F737461745F6C6F672F3330303030313031543030303030302E3030305A5F3330303030313032543030303030302E3030305A2F323032312D30372D30325430325F34335F30392E3631355A2F305F696E6465782E7A6970227D2C2264696D656E73696F6E73223A226170705F6E616D652C6332735F627974655F6E756D2C6332735F706B745F6E756D2C7332635F627974655F6E756D2C7332635F706B745F6E756D2C73657373696F6E5F6E756D2C7375625F6170705F6E616D65222C226D657472696373223A22222C22736861726453706563223A7B2274797065223A226E756D6265726564222C22706172746974696F6E4E756D223A302C22706172746974696F6E73223A307D2C2262696E61727956657273696F6E223A392C2273697A65223A323139352C226964656E746966696572223A22747261666669635F6170705F737461745F6C6F675F333030302D30312D30325430303A30303A30302E3030305A5F333030302D30312D30335430303A30303A30302E3030305A5F323032312D30372D30325430323A34333A30392E3631355A227D); + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/initData.sh b/parcels/roles/init/druid/files/druid_topology/initDruidData/initData.sh new file mode 100755 index 0000000..dd20185 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/initData.sh @@ -0,0 +1,15 @@ +#!/bin/bash +PWD=`pwd` +task_name=`cat ${PWD}/tasks.txt` +#overlord节点IP +overlord_ip=$1 + + +for var in ${task_name[@]}; +do +curl -X 'POST' -H 'Content-Type:application/json' -d @${var} http://${overlord_ip}:8081/druid/indexer/v1/task +echo "'${var}' 数据初始化成功" +sleep 2 +done + + diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_client_internal_ip_hot_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_client_internal_ip_hot_log.json new file mode 100644 index 0000000..0555054 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_client_internal_ip_hot_log.json @@ -0,0 +1,70 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"common_client_ip\":\" \",\"common_internal_ip\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"common_l4_protocol\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "client_internal_ip_hot_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "common_client_ip", + "type": "string" + }, + { + "name": "common_internal_ip", + "type": "string" + }, + { + "name": "common_l4_protocol", + "type": "string" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + } + ] + } + } + } + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_proxy_event_hits_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_proxy_event_hits_log.json new file mode 100644 index 0000000..64d929a --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_proxy_event_hits_log.json @@ -0,0 +1,85 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"country\":\" \",\"action\":1024,\"c2s_byte_num\":0,\"s2c_byte_num\":0,\"entrance_id\":0,\"policy_id\":0,\"location\":\" \",\"hits\":0,\"sub_action\":\" \",\"ips\":\" \",\"isp\":\" \"}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "proxy_event_hits_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": true + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "country", + "type": "string" + }, + { + "name": "isp", + "type": "string" + }, + { + "name": "sub_action", + "type": "string" + }, + { + "name": "location", + "type": "string" + }, + { + "name": "action", + "type": "long" + }, + { + "name": "entrance_id", + "type": "long" + }, + { + "name": "hits", + "type": "long" + }, + { + "name": "policy_id", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + } + ] + } + } + }, + "metricsSpec": [ + { + "type": "HLLSketchBuild", + "name": "ip_object", + "fieldName": "ips" + } + ] + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_proxy_event_urls_hot_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_proxy_event_urls_hot_log.json new file mode 100644 index 0000000..7dfdd97 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_proxy_event_urls_hot_log.json @@ -0,0 +1,46 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"url\":\" \",\"session_num\":0,\"device_id\":\" \",\"ips\":\" \"}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "proxy_event_urls_hot_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "url", + "type": "string" + }, + { + "name": "session_num", + "type": "long" + } + ] + } + } + } + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_security_event_hits_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_security_event_hits_log.json new file mode 100644 index 0000000..e503cba --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_security_event_hits_log.json @@ -0,0 +1,66 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"country\":\" \",\"action\":1024,\"c2s_byte_num\":0,\"s2c_byte_num\":0,\"entrance_id\":0,\"policy_id\":0,\"location\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "security_event_hits_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "isp", + "type": "string" + }, + { + "name": "action", + "type": "long" + }, + { + "name": "entrance_id", + "type": "long" + }, + { + "name": "hits", + "type": "long" + }, + { + "name": "policy_id", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + } + ] + } + } + } + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_security_event_urls_hot_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_security_event_urls_hot_log.json new file mode 100644 index 0000000..20fda07 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_security_event_urls_hot_log.json @@ -0,0 +1,46 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"url\":\" \",\"session_num\":0,\"device_id\":\" \",\"ips\":\" \"}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "security_event_urls_hot_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "url", + "type": "string" + }, + { + "name": "session_num", + "type": "long" + } + ] + } + } + } + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_server_external_ip_hot_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_server_external_ip_hot_log.json new file mode 100644 index 0000000..e79b15d --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_server_external_ip_hot_log.json @@ -0,0 +1,70 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"common_server_ip\":\" \",\"common_external_ip\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"common_l4_protocol\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "server_external_ip_hot_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "common_server_ip", + "type": "string" + }, + { + "name": "common_external_ip", + "type": "string" + }, + { + "name": "common_l4_protocol", + "type": "string" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + } + ] + } + } + } + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_sys_storage_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_sys_storage_log.json new file mode 100644 index 0000000..8b88307 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_sys_storage_log.json @@ -0,0 +1,62 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"log_type\":\" \",\"data_center\":\" \",\"aggregate_size\":0,\"first_storage\":0,\"max_size\":0,\"stored_days\":0,\"used_size\":0,\"location\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "sys_storage_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "log_type", + "type": "string" + }, + { + "name": "data_center", + "type": "string" + }, + { + "name": "aggregate_size", + "type": "long" + }, + { + "name": "last_storage", + "type": "long" + }, + { + "name": "max_size", + "type": "long" + }, + { + "name": "used_size", + "type": "long" + } + ] + } + } + } + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_client_ip_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_client_ip_log.json new file mode 100644 index 0000000..86027b3 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_client_ip_log.json @@ -0,0 +1,66 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"source\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "top_client_ip_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "source", + "type": "string" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "order_by", + "type": "string" + } + ] + } + } + } + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_external_host_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_external_host_log.json new file mode 100644 index 0000000..fb54545 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_external_host_log.json @@ -0,0 +1,66 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"destination\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "top_external_host_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "destination", + "type": "string" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "order_by", + "type": "string" + } + ] + } + } + } + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_internal_host_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_internal_host_log.json new file mode 100644 index 0000000..8678bbd --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_internal_host_log.json @@ -0,0 +1,66 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"source\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "top_internal_host_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "source", + "type": "string" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "order_by", + "type": "string" + } + ] + } + } + } + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_server_ip_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_server_ip_log.json new file mode 100644 index 0000000..4358106 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_server_ip_log.json @@ -0,0 +1,66 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"destination\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "top_server_ip_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "destination", + "type": "string" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "order_by", + "type": "string" + } + ] + } + } + } + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_urls_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_urls_log.json new file mode 100644 index 0000000..9a9c7d2 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_urls_log.json @@ -0,0 +1,46 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"url\":\" \",\"session_num\":0,\"device_id\":\" \",\"ips\":\" \"}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "top_urls_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "url", + "type": "string" + }, + { + "name": "session_num", + "type": "long" + } + ] + } + } + } + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_user_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_user_log.json new file mode 100644 index 0000000..61aaa46 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_user_log.json @@ -0,0 +1,66 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"subscriber_id\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "top_user_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "subscriber_id", + "type": "string" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "order_by", + "type": "string" + } + ] + } + } + } + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_website_domain_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_website_domain_log.json new file mode 100644 index 0000000..828a7c6 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_top_website_domain_log.json @@ -0,0 +1,66 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"domain\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "top_website_domain_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "domain", + "type": "string" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "order_by", + "type": "string" + } + ] + } + } + } + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_app_stat_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_app_stat_log.json new file mode 100644 index 0000000..c4e2a64 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_app_stat_log.json @@ -0,0 +1,67 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"app_name\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"sub_app_name\":\" \"}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "traffic_app_stat_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "app_name", + "type": "string" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "sub_app_name", + "type": "string" + } + ] + } + } + } + } + } +} + diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_metrics_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_metrics_log.json new file mode 100644 index 0000000..b530649 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_metrics_log.json @@ -0,0 +1,88 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"device_id\":\" \",\"allow_conn_num\":0,\"allow_in_bytes\":0,\"allow_in_packets\":0,\"allow_out_bytes\":0,\"allow_out_packets\":0,\"close_conn_num\":0,\"default_conn_num\":0,\"default_in_bytes\":0,\"default_in_packets\":0,\"default_out_bytes\":0,\"default_out_packets\":0,\"deny_conn_num\":0,\"deny_in_bytes\":0,\"deny_in_packets\":0,\"deny_out_bytes\":0,\"deny_out_packets\":0,\"established_conn_num\":0,\"intercept_conn_num\":0,\"intercept_in_bytes\":0,\"intercept_in_packets\":0,\"intercept_out_bytes\":0,\"intercept_out_packets\":0,\"monitor_conn_num\":0,\"monitor_in_bytes\":0,\"monitor_in_packets\":0,\"monitor_out_bytes\":0,\"monitor_out_packets\":0,\"new_conn_num\":0,\"total_in_bytes\":0,\"total_in_packets\":0,\"total_out_bytes\":0,\"total_out_packets\":0,\"entrance_id\":0,\"pinning_num\":0,\"not_pinning_num\":0,\"maybe_pinning_num\":0,\"alert_bytes\":0,\"ins_bytes\":0,\"hijk_bytes\":0,\"reflection_bytes\":0,\"flood_bytes\":0,\"intcp_mon_num\":0,\"intcp_allow_num\":0,\"intcp_deny_num\":0,\"intcp_rdirt_num\":0,\"intcp_repl_num\":0,\"intcp_hijk_num\":0,\"intcp_ins_num\":0,\"ad_reflection_bytes\":0,\"ad_flood_bytes\":0,\"ad_cc_bytes\":0}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "traffic_metrics_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "device_id","type": "string"}, + {"name": "allow_conn_num","type": "long"}, + {"name": "allow_in_bytes","type": "long"}, + {"name": "allow_in_packets","type": "long"}, + {"name": "allow_out_bytes","type": "long"}, + {"name": "allow_out_packets","type": "long"}, + {"name": "close_conn_num","type": "long"}, + {"name": "default_conn_num","type": "long"}, + {"name": "default_in_bytes","type": "long"}, + {"name": "default_in_packets","type": "long"}, + {"name": "default_out_bytes","type": "long"}, + {"name": "default_out_packets","type": "long"}, + {"name": "deny_conn_num","type": "long"}, + {"name": "deny_in_bytes","type": "long"}, + {"name": "deny_in_packets","type": "long"}, + {"name": "deny_out_bytes","type": "long"}, + {"name": "deny_out_packets","type": "long"}, + {"name": "established_conn_num","type": "long"}, + {"name": "intercept_conn_num","type": "long"}, + {"name": "intercept_in_bytes","type": "long"}, + {"name": "intercept_in_packets","type": "long"}, + {"name": "intercept_out_bytes","type": "long"}, + {"name": "intercept_out_packets","type": "long"}, + {"name": "monitor_conn_num","type": "long"}, + {"name": "monitor_in_bytes","type": "long"}, + {"name": "monitor_in_packets","type": "long"}, + {"name": "monitor_out_bytes","type": "long"}, + {"name": "monitor_out_packets","type": "long"}, + {"name": "new_conn_num","type": "long"}, + {"name": "total_in_bytes","type": "long"}, + {"name": "total_in_packets","type": "long"}, + {"name": "total_out_bytes","type": "long"}, + {"name": "entrance_id","type": "long"}, + {"name": "total_out_packets","type": "long"}, + {"name": "pinning_num","type": "long"}, + {"name": "not_pinning_num","type": "long"}, + {"name": "maybe_pinning_num","type": "long"}, + {"name": "alert_bytes","type": "long"}, + {"name": "ins_bytes","type": "long"}, + {"name": "hijk_bytes","type": "long"}, + {"name": "ad_reflection_bytes","type": "long"}, + {"name": "ad_flood_bytes","type": "long"}, + {"name": "ad_cc_bytes","type": "long"}, + {"name": "intcp_mon_num","type": "long"}, + {"name": "intcp_allow_num","type": "long"}, + {"name": "intcp_deny_num","type": "long"}, + {"name": "intcp_rdirt_num","type": "long"}, + {"name": "intcp_repl_num","type": "long"}, + {"name": "intcp_hijk_num","type": "long"}, + {"name": "intcp_ins_num","type": "long"} + ] + } + } + } + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_protocol_stat_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_protocol_stat_log.json new file mode 100644 index 0000000..aa56c33 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_protocol_stat_log.json @@ -0,0 +1,54 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"protocol_id\":\" \",\"country\":\" \",\"one_sided_connections\":1024,\"sequence_gap_loss\":0,\"s2c_byte_num\":0,\"entrance_id\":0,\"fragmentation_packets\":0,\"location\":\" \",\"uncategorized_bytes\":0,\"unorder_packets\":0,\"sub_action\":\" \",\"ips\":\" \",\"isp\":\" \",\"schema_type\":\" \",\"data_center\":\" \",\"s2c_pkt_num\":0,\"s2c_pkt_num\":0,\"c2s_ipfrag_num\":0,\"s2c_ipfrag_num\":0,\"c2s_tcp_lostlen\":0,\"s2c_tcp_lostlen\":0,\"c2s_tcp_unorder_num\":0,\"s2c_tcp_unorder_num\":0}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "traffic_protocol_stat_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": true + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "isp","type": "string"}, + {"name": "protocol_id","type": "string"}, + {"name": "data_center","type": "string"}, + {"name": "c2s_pkt_num","type": "long"}, + {"name": "entrance_id","type": "long"}, + {"name": "sessions","type": "long"}, + {"name": "s2c_pkt_num","type": "long"}, + {"name": "c2s_byte_num","type": "long"}, + {"name": "s2c_byte_num","type": "long"}, + {"name": "c2s_ipfrag_num","type": "long"}, + {"name": "s2c_ipfrag_num","type": "long"}, + {"name": "c2s_tcp_lostlen","type": "long"}, + {"name": "s2c_tcp_lostlen","type": "long"}, + {"name": "c2s_tcp_unorder_num","type": "long"}, + {"name": "s2c_tcp_unorder_num","type": "long"} + ] + } + } + }, + "metricsSpec": [] + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_summary_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_summary_log.json new file mode 100644 index 0000000..582beb4 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_traffic_summary_log.json @@ -0,0 +1,53 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"country\":\" \",\"one_sided_connections\":1024,\"sequence_gap_loss\":0,\"s2c_byte_num\":0,\"entrance_id\":0,\"fragmentation_packets\":0,\"location\":\" \",\"uncategorized_bytes\":0,\"unorder_packets\":0,\"sub_action\":\" \",\"ips\":\" \",\"isp\":\" \",\"schema_type\":\" \",\"data_center\":\" \",\"s2c_pkt_num\":0,\"s2c_pkt_num\":0,\"c2s_ipfrag_num\":0,\"s2c_ipfrag_num\":0,\"c2s_tcp_lostlen\":0,\"s2c_tcp_lostlen\":0,\"c2s_tcp_unorder_num\":0,\"s2c_tcp_unorder_num\":0}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "traffic_summary_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": true + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + {"name": "isp","type": "string"}, + {"name": "schema_type","type": "string"}, + {"name": "data_center","type": "string"}, + {"name": "c2s_pkt_num","type": "long"}, + {"name": "entrance_id","type": "long"}, + {"name": "sessions","type": "long"}, + {"name": "s2c_pkt_num","type": "long"}, + {"name": "c2s_byte_num","type": "long"}, + {"name": "s2c_byte_num","type": "long"}, + {"name": "one_sided_connections","type": "long"}, + {"name": "uncategorized_bytes","type": "long"}, + {"name": "fragmentation_packets","type": "long"}, + {"name": "sequence_gap_loss","type": "long"}, + {"name": "unorder_packets","type": "long"} + ] + } + } + }, + "metricsSpec": [{"type": "HLLSketchBuild","name": "ip_object","fieldName": "ips"}] + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_user_hot_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_user_hot_log.json new file mode 100644 index 0000000..9f99cd3 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_user_hot_log.json @@ -0,0 +1,62 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"common_subscriber_id\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "user_hot_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "common_subscriber_id", + "type": "string" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + } + ] + } + } + } + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/init_website_domain_hot_log.json b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_website_domain_hot_log.json new file mode 100644 index 0000000..e9df8bd --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/init_website_domain_hot_log.json @@ -0,0 +1,61 @@ +{ + "type": "index", + "spec":{ + "ioConfig": { + "type": "index", + "firehose": { + "type": "inline", + "data": "{\"test_time\":\"3000-01-01T00:00:00.000\",\"domain\":\" \",\"c2s_byte_num\":0,\"c2s_pkt_num\":0,\"s2c_byte_num\":0,\"s2c_pkt_num\":0,\"session_num\":0,\"order_by\":\" \",\"hits\":0,\"sub_action\":\" \",\"intercept_state\":0,\"pinningst\":0,\"ips\":\" \",\"isp\":\" \"}" + } + }, + "tuningConfig": { + "type": "index" + }, + "dataSchema": { + "dataSource": "website_domain_hot_log", + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": "NONE", + "rollup": false + }, + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "test_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + { + "name": "domain", + "type": "string" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + }]} + }, + "metricsSpec": [{"type": "HLLSketchBuild","name": "ip_object","fieldName": "ips"}] + } + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/initDruidData/tasks.txt b/parcels/roles/init/druid/files/druid_topology/initDruidData/tasks.txt new file mode 100644 index 0000000..791a829 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/initDruidData/tasks.txt @@ -0,0 +1,20 @@ +init_client_internal_ip_hot_log.json +init_proxy_event_hits_log.json +init_proxy_event_urls_hot_log.json +init_security_event_hits_log.json +init_security_event_urls_hot_log.json +init_server_external_ip_hot_log.json +init_sys_storage_log.json +init_top_client_ip_log.json +init_top_external_host_log.json +init_top_internal_host_log.json +init_top_server_ip_log.json +init_top_urls_log.json +init_top_user_log.json +init_top_website_domain_log.json +init_traffic_metrics_log.json +init_traffic_protocol_stat_log.json +init_traffic_summary_log.json +init_user_hot_log.json +init_website_domain_hot_log.json +init_traffic_app_stat_log.json diff --git a/parcels/roles/init/druid/files/druid_topology/rule/common_data_rule.json b/parcels/roles/init/druid/files/druid_topology/rule/common_data_rule.json new file mode 100644 index 0000000..ad1c808 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/rule/common_data_rule.json @@ -0,0 +1,13 @@ +[ + { + "type" : "loadByPeriod", + "period" : "P365D", + "includeFuture" : true, + "tieredReplicants": { + "_default_tier" : 1 + } + }, + { + "type" : "dropForever" + } +] diff --git a/parcels/roles/init/druid/files/druid_topology/rule/common_task.txt b/parcels/roles/init/druid/files/druid_topology/rule/common_task.txt new file mode 100644 index 0000000..fe803ab --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/rule/common_task.txt @@ -0,0 +1,13 @@ +traffic_summary_log +proxy_event_hits_log +security_event_hits_log +top_client_ip_log +top_server_ip_log +top_internal_host_log +top_external_host_log +top_website_domain_log +top_user_log +top_urls_log +traffic_protocol_stat_log +traffic_metrics_log +sys_storage_log diff --git a/parcels/roles/init/druid/files/druid_topology/rule/hot_data_rule.json b/parcels/roles/init/druid/files/druid_topology/rule/hot_data_rule.json new file mode 100644 index 0000000..a44486e --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/rule/hot_data_rule.json @@ -0,0 +1,13 @@ +[ + { + "type" : "loadByPeriod", + "period" : "PT1H", + "includeFuture" : true, + "tieredReplicants": { + "_default_tier" : 1 + } + }, + { + "type" : "dropForever" + } +] diff --git a/parcels/roles/init/druid/files/druid_topology/rule/hot_task.txt b/parcels/roles/init/druid/files/druid_topology/rule/hot_task.txt new file mode 100644 index 0000000..7dcddc6 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/rule/hot_task.txt @@ -0,0 +1,6 @@ +proxy_event_urls_hot_log +security_event_urls_hot_log +client_internal_ip_hot_log +server_external_ip_hot_log +top_user_log_hot +top_website_domain_log_hot diff --git a/parcels/roles/init/druid/files/druid_topology/rule/supervisor-manger b/parcels/roles/init/druid/files/druid_topology/rule/supervisor-manger new file mode 100644 index 0000000..33673cf --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/rule/supervisor-manger @@ -0,0 +1,77 @@ +#!/bin/bash + +base_dir=`dirname $0` +base_dir="$(cd "$base_dir" && pwd)" +#echo $base_dir + +#router_ip='10.4.61.201' +router_ip='192.168.44.10' + +common_task_name=`cat $base_dir/common_task.txt` +common_rule_file="$base_dir/common_data_rule.json" + +hot_task_name=`cat $base_dir/hot_task.txt` +hot_rule_file="$base_dir/hot_data_rule.json" + +case $1 in + + resetAll) + for var in ${common_task_name[@]}; + do + curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/${var}/reset + sleep 2 + done + + for var in ${hot_task_name[@]}; + do + curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/${var}/reset + sleep 2 + done + ;; + + terminateAll) + curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/terminateAll + ;; + + shutdownAllTasks) + for var in ${common_task_name[@]}; + do + curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/${var}/shutdownAllTasks + sleep 2 + done + + for var in ${hot_task_name[@]}; + do + curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/${var}/shutdownAllTasks + sleep 2 + done + ;; + + reset) + if [ $# -le 1 ]; then + echo "Usage: supervisor-manger reset <dataSource>" + exit 1 + fi + curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/$2/reset + ;; + + terminate) + if [ $# -le 1 ]; then + echo "Usage: supervisor-manger terminate <dataSource>" + exit 1 + fi + curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/$2/terminate + ;; + + shutdownTasks) + if [ $# -le 1 ]; then + echo "Usage: supervisor-manger shutdownTasks <dataSource>" + exit 1 + fi + curl -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/$2/shutdownAllTasks + ;; + *) + echo "Usage: supervisor-manger {resetAll|terminateAll|shutdownAllTasks}" + echo "Usage: supervisor-manger {reset|terminate|shutdownTasks} <dataSource>" + ;; +esac diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/client_internal_ip_hot_log.json b/parcels/roles/init/druid/files/druid_topology/tasks/client_internal_ip_hot_log.json new file mode 100644 index 0000000..3d3829c --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/client_internal_ip_hot_log.json @@ -0,0 +1,54 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "client_internal_ip_hot_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "common_recv_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "common_internal_ip", + "common_client_ip", + "common_l4_protocol" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num","fieldName" : "common_sessions"}, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "common_c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "common_s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[] + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "CONNECTION-RECORD-COMPLETED-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "earlyMessageRejectionPeriod": "PT6H", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/proxy_event_hits_log.json b/parcels/roles/init/druid/files/druid_topology/tasks/proxy_event_hits_log.json new file mode 100644 index 0000000..936c539 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/proxy_event_hits_log.json @@ -0,0 +1,94 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "proxy_event_hits_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "common_recv_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + { "name": "policy_id", "type": "long" }, + "isp", + "country", + "location", + { "name": "entrance_id", "type": "long" }, + { "name": "action", "type": "long" }, + "sub_action" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "HLLSketchBuild", "name" : "ip_object", "fieldName" : "common_client_ip" }, + { "type" : "count", "name" : "hits" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + { + "type": "expression", + "name": "country", + "expression": "array_ordinal(string_to_array(common_client_location,','),3)" + }, + { + "type": "expression", + "name": "location", + "expression": "array_ordinal(string_to_array(common_client_location,','),2)" + }, + { + "type": "expression", + "name": "policy_id", + "expression": "common_policy_id" + }, + { + "type": "expression", + "name": "isp", + "expression": "common_isp" + }, + { + "type": "expression", + "name": "entrance_id", + "expression": "common_entrance_id" + }, + { + "type": "expression", + "name": "sub_action", + "expression": "common_sub_action" + }, + { + "type": "expression", + "name": "action", + "expression": "common_action" + } + ] + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 20000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "PROXY-EVENT-COMPLETED-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/security_event_hits_log.json b/parcels/roles/init/druid/files/druid_topology/tasks/security_event_hits_log.json new file mode 100644 index 0000000..c72a6a8 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/security_event_hits_log.json @@ -0,0 +1,74 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "security_event_hits_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "common_recv_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + { "name": "policy_id", "type": "long" }, + "isp", + { "name": "entrance_id", "type": "long" }, + { "name": "action", "type": "long" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "count", "name" : "hits" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT1M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + { + "type": "expression", + "name": "policy_id", + "expression": "common_policy_id" + }, + { + "type": "expression", + "name": "isp", + "expression": "common_isp" + }, + { + "type": "expression", + "name": "entrance_id", + "expression": "common_entrance_id" + }, + { + "type": "expression", + "name": "action", + "expression": "common_action" + } + ] + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "SECURITY-EVENT-COMPLETED-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/server_external_ip_hot_log.json b/parcels/roles/init/druid/files/druid_topology/tasks/server_external_ip_hot_log.json new file mode 100644 index 0000000..34b8a63 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/server_external_ip_hot_log.json @@ -0,0 +1,54 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "server_external_ip_hot_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "common_recv_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "common_external_ip", + "common_server_ip", + "common_l4_protocol" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num","fieldName" : "common_sessions"}, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "common_c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "common_s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[] + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "CONNECTION-RECORD-COMPLETED-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "earlyMessageRejectionPeriod": "PT6H", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/sys_storage_log.json b/parcels/roles/init/druid/files/druid_topology/tasks/sys_storage_log.json new file mode 100644 index 0000000..22b1b48 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/sys_storage_log.json @@ -0,0 +1,49 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "sys_storage_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "log_type", + "data_center", + { "name": "used_size", "type": "long" }, + { "name": "max_size", "type": "long" }, + { "name": "aggregate_size", "type": "long" }, + { "name": "last_storage", "type": "long" } + ] + } + } + }, + "metricsSpec" : [ + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": false + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "SYS-STORAGE-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/top_client_ip_log.json b/parcels/roles/init/druid/files/druid_topology/tasks/top_client_ip_log.json new file mode 100644 index 0000000..c442ef0 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/top_client_ip_log.json @@ -0,0 +1,50 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_client_ip_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "stat_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + "source", + "order_by" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" }, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TOP-CLIENT-IP-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/top_external_host_log.json b/parcels/roles/init/druid/files/druid_topology/tasks/top_external_host_log.json new file mode 100644 index 0000000..dee41ad --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/top_external_host_log.json @@ -0,0 +1,51 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_external_host_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "stat_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + "destination", + "order_by" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" }, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TOP-EXTERNAL-HOST-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/top_internal_host_log.json b/parcels/roles/init/druid/files/druid_topology/tasks/top_internal_host_log.json new file mode 100644 index 0000000..928fd7a --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/top_internal_host_log.json @@ -0,0 +1,50 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_internal_host_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "stat_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + "source", + "order_by" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" }, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TOP-INTERNAL-HOST-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/top_server_ip_log.json b/parcels/roles/init/druid/files/druid_topology/tasks/top_server_ip_log.json new file mode 100644 index 0000000..3b2d84a --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/top_server_ip_log.json @@ -0,0 +1,51 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_server_ip_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "stat_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + "destination", + "order_by" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" }, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TOP-SERVER-IP-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/top_urls_log.json b/parcels/roles/init/druid/files/druid_topology/tasks/top_urls_log.json new file mode 100644 index 0000000..849196b --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/top_urls_log.json @@ -0,0 +1,47 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_urls_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "stat_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + "url" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{} + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TOP-URLS-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/top_urls_log_proxy_hot.json b/parcels/roles/init/druid/files/druid_topology/tasks/top_urls_log_proxy_hot.json new file mode 100644 index 0000000..1050fa1 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/top_urls_log_proxy_hot.json @@ -0,0 +1,59 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "proxy_event_urls_hot_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "common_recv_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "url" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "count", "name" : "session_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + { + "type": "expression", + "name": "url", + "expression": "http_url" + } + ], + "filter": { + "type": "not", + "field":{ "type": "selector", "dimension": "url", "value": "" } + } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "PROXY-EVENT-COMPLETED-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/top_urls_log_security_hot.json b/parcels/roles/init/druid/files/druid_topology/tasks/top_urls_log_security_hot.json new file mode 100644 index 0000000..1e832eb --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/top_urls_log_security_hot.json @@ -0,0 +1,59 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "security_event_urls_hot_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "common_recv_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "url" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "count", "name" : "session_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + { + "type": "expression", + "name": "url", + "expression": "http_url" + } + ], + "filter": { + "type": "not", + "field":{ "type": "selector", "dimension": "url", "value": "" } + } + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "SECURITY-EVENT-COMPLETED-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/top_user_log.json b/parcels/roles/init/druid/files/druid_topology/tasks/top_user_log.json new file mode 100644 index 0000000..3e3d243 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/top_user_log.json @@ -0,0 +1,50 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_user_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "stat_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + "subscriber_id", + "order_by" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" }, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TOP-USER-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/top_user_log_hot.json b/parcels/roles/init/druid/files/druid_topology/tasks/top_user_log_hot.json new file mode 100644 index 0000000..9f0b7d1 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/top_user_log_hot.json @@ -0,0 +1,56 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "user_hot_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "common_recv_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "common_subscriber_id" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num","fieldName" : "common_sessions"}, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "common_c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "common_s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[], + "filter": { + "type": "not", + "field":{ "type": "selector", "dimension": "common_subscriber_id", "value": "" } + } + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "CONNECTION-RECORD-COMPLETED-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/top_website_domain_log.json b/parcels/roles/init/druid/files/druid_topology/tasks/top_website_domain_log.json new file mode 100644 index 0000000..cbcaa94 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/top_website_domain_log.json @@ -0,0 +1,51 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "top_website_domain_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "stat_time", + "format": "iso" + }, + "dimensionsSpec": { + "dimensions": [ + "domain", + "order_by" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num", "fieldName" : "session_num" }, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TOP-WEBSITE-DOMAIN-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/top_website_domain_log_hot.json b/parcels/roles/init/druid/files/druid_topology/tasks/top_website_domain_log_hot.json new file mode 100644 index 0000000..29a3cbd --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/top_website_domain_log_hot.json @@ -0,0 +1,69 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "website_domain_hot_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "common_recv_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "domain", + "protocol_type" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num","fieldName" : "common_sessions"}, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "common_c2s_pkt_num" }, + {"type": "HLLSketchBuild","name": "ip_object","fieldName": "common_client_ip","lgK": 12,"tgtHllType": "HLL_4","round": false}, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "common_s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + { + "type": "expression", + "name": "protocol_type", + "expression": "common_schema_type" + }, + { + "type": "expression", + "name": "domain", + "expression": "http_domain" + } + ], + "filter": { + "type": "not", + "field":{ "type": "selector", "dimension": "domain", "value": "" } + } + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "CONNECTION-RECORD-COMPLETED-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/traffic_app_stat_log.json b/parcels/roles/init/druid/files/druid_topology/tasks/traffic_app_stat_log.json new file mode 100644 index 0000000..e630acb --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/traffic_app_stat_log.json @@ -0,0 +1,61 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "traffic_app_stat_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "common_recv_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "app_name", + "sub_app_name" + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "session_num","fieldName" : "common_sessions"}, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "common_c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "common_s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression", "name": "app_name", "expression": "array_ordinal(string_to_array(replace(common_app_label, '.', '/'),'/'),1)"}, + {"type": "expression", "name": "sub_app_name", "expression": "array_ordinal(string_to_array(replace(common_app_label, '.', '/'),'/'),2)"} + ], + "filter": { + "type": "not", + "field":{ "type": "selector", "dimension": "app_name", "value": "" } + } + } + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "CONNECTION-RECORD-COMPLETED-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "earlyMessageRejectionPeriod": "PT6H", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap" + } + } +} + diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/traffic_metrics_log.json b/parcels/roles/init/druid/files/druid_topology/tasks/traffic_metrics_log.json new file mode 100644 index 0000000..76afdd9 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/traffic_metrics_log.json @@ -0,0 +1,150 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "traffic_metrics_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "timestamp", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "device_id", + "entrance_id" + ] + }, + "flattenSpec": { + "useFieldDiscovery": true, + "fields": [ + { "name": "allow_conn_num", "type": "path", "expr": "$.fields.allow_conn_num" }, + { "name": "allow_in_bytes", "type": "path", "expr": "$.fields.allow_in_bytes" }, + { "name": "allow_in_packets", "type": "path", "expr": "$.fields.allow_in_packets" }, + { "name": "allow_out_bytes", "type": "path", "expr": "$.fields.allow_out_bytes" }, + { "name": "allow_out_packets", "type": "path", "expr": "$.fields.allow_out_packets" }, + { "name": "close_conn_num", "type": "path", "expr": "$.fields.close_conn_num" }, + { "name": "default_conn_num", "type": "path", "expr": "$.fields.default_conn_num" }, + { "name": "default_in_bytes", "type": "path", "expr": "$.fields.default_in_bytes" }, + { "name": "default_in_packets", "type": "path", "expr": "$.fields.default_in_packets" }, + { "name": "default_out_bytes", "type": "path", "expr": "$.fields.default_out_bytes" }, + { "name": "default_out_packets", "type": "path", "expr": "$.fields.default_out_packets" }, + { "name": "deny_conn_num", "type": "path", "expr": "$.fields.deny_conn_num" }, + { "name": "deny_in_bytes", "type": "path", "expr": "$.fields.deny_in_bytes" }, + { "name": "deny_in_packets", "type": "path", "expr": "$.fields.deny_in_packets" }, + { "name": "deny_out_bytes", "type": "path", "expr": "$.fields.deny_out_bytes" }, + { "name": "deny_out_packets", "type": "path", "expr": "$.fields.deny_out_packets" }, + { "name": "established_conn_num", "type": "path", "expr": "$.fields.established_conn_num" }, + { "name": "intercept_conn_num", "type": "path", "expr": "$.fields.intercept_conn_num" }, + { "name": "intercept_in_bytes", "type": "path", "expr": "$.fields.intercept_in_bytes" }, + { "name": "intercept_in_packets", "type": "path", "expr": "$.fields.intercept_in_packets" }, + { "name": "intercept_out_bytes", "type": "path", "expr": "$.fields.intercept_out_bytes" }, + { "name": "intercept_out_packets", "type": "path", "expr": "$.fields.intercept_out_packets" }, + { "name": "monitor_conn_num", "type": "path", "expr": "$.fields.monitor_conn_num" }, + { "name": "monitor_in_bytes", "type": "path", "expr": "$.fields.monitor_in_bytes" }, + { "name": "monitor_in_packets", "type": "path", "expr": "$.fields.monitor_in_packets" }, + { "name": "monitor_out_bytes", "type": "path", "expr": "$.fields.monitor_out_bytes" }, + { "name": "monitor_out_packets", "type": "path", "expr": "$.fields.monitor_out_packets" }, + { "name": "new_conn_num", "type": "path", "expr": "$.fields.new_conn_num" }, + { "name": "total_in_bytes", "type": "path", "expr": "$.fields.total_in_bytes" }, + { "name": "total_in_packets", "type": "path", "expr": "$.fields.total_in_packets" }, + { "name": "total_out_bytes", "type": "path", "expr": "$.fields.total_out_bytes" }, + { "name": "total_out_packets", "type": "path", "expr": "$.fields.total_out_packets" }, + { "name": "pinning_num", "type": "path", "expr": "$.fields.pinning_num" }, + { "name": "not_pinning_num", "type": "path", "expr": "$.fields.not_pinning_num" }, + { "name": "maybe_pinning_num", "type": "path", "expr": "$.fields.maybe_pinning_num" }, + { "name": "alert_bytes", "type": "path", "expr": "$.fields.alert_bytes" }, + { "name": "insert_bytes", "type": "path", "expr": "$.fields.insert_bytes" }, + { "name": "hijack_bytes", "type": "path", "expr": "$.fields.hijack_bytes" }, + { "name": "ad_reflection_bytes", "type": "path", "expr": "$.fields.ad_reflection_bytes" }, + { "name": "ad_flood_bytes", "type": "path", "expr": "$.fields.ad_flood_bytes" }, + { "name": "ad_cc_bytes", "type": "path", "expr": "$.fields.ad_cc_bytes" }, + { "name": "intercept_monitor_conn_num", "type": "path", "expr": "$.fields.intcp_mon_num" }, + { "name": "intercept_allow_conn_num", "type": "path", "expr": "$.fields.intcp_allow_num" }, + { "name": "intercept_deny_conn_num", "type": "path", "expr": "$.fields.intcp_deny_num" }, + { "name": "intercept_redirect_conn_num", "type": "path", "expr": "$.fields.intcp_rdirt_num" }, + { "name": "intercept_replace_conn_num", "type": "path", "expr": "$.fields.intcp_repl_num" }, + { "name": "intercept_hijack_conn_num", "type": "path", "expr": "$.fields.intcp_hijk_num" }, + { "name": "intercept_insert_conn_num", "type": "path", "expr": "$.fields.intcp_ins_num" }, + { "name": "app_name", "type": "path", "expr": "$.tags.app_name" }, + { "name": "device_id", "type": "path", "expr": "$.tags.device_id" }, + { "name": "entrance_id", "type": "path", "expr": "$.tags.entrance_id" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "new_conn_num", "fieldName" : "new_conn_num" }, + { "type" : "longMax", "name" : "established_conn_num", "fieldName" : "established_conn_num" }, + { "type" : "longSum", "name" : "close_conn_num", "fieldName" : "close_conn_num" }, + { "type" : "longSum", "name" : "total_in_bytes", "fieldName" : "total_in_bytes" }, + { "type" : "longSum", "name" : "total_out_bytes", "fieldName" : "total_out_bytes" }, + { "type" : "longSum", "name" : "total_in_packets", "fieldName" : "total_in_packets" }, + { "type" : "longSum", "name" : "total_out_packets", "fieldName" : "total_out_packets" }, + { "type" : "longSum", "name" : "default_conn_num", "fieldName" : "default_conn_num" }, + { "type" : "longSum", "name" : "default_in_bytes", "fieldName" : "default_in_bytes" }, + { "type" : "longSum", "name" : "default_out_bytes", "fieldName" : "default_out_bytes" }, + { "type" : "longSum", "name" : "default_in_packets", "fieldName" : "default_in_packets" }, + { "type" : "longSum", "name" : "default_out_packets", "fieldName" : "default_out_packets" }, + { "type" : "longSum", "name" : "allow_conn_num", "fieldName" : "allow_conn_num" }, + { "type" : "longSum", "name" : "allow_in_bytes", "fieldName" : "allow_in_bytes" }, + { "type" : "longSum", "name" : "allow_out_bytes", "fieldName" : "allow_out_bytes" }, + { "type" : "longSum", "name" : "allow_in_packets", "fieldName" : "allow_in_packets" }, + { "type" : "longSum", "name" : "allow_out_packets", "fieldName" : "allow_out_packets" }, + { "type" : "longSum", "name" : "deny_conn_num", "fieldName" : "deny_conn_num" }, + { "type" : "longSum", "name" : "deny_in_bytes", "fieldName" : "deny_in_bytes" }, + { "type" : "longSum", "name" : "deny_out_bytes", "fieldName" : "deny_out_bytes" }, + { "type" : "longSum", "name" : "deny_in_packets", "fieldName" : "deny_in_packets" }, + { "type" : "longSum", "name" : "deny_out_packets", "fieldName" : "deny_out_packets" }, + { "type" : "longSum", "name" : "monitor_conn_num", "fieldName" : "monitor_conn_num" }, + { "type" : "longSum", "name" : "monitor_in_bytes", "fieldName" : "monitor_in_bytes" }, + { "type" : "longSum", "name" : "monitor_out_bytes", "fieldName" : "monitor_out_bytes" }, + { "type" : "longSum", "name" : "monitor_in_packets", "fieldName" : "monitor_in_packets" }, + { "type" : "longSum", "name" : "monitor_out_packets", "fieldName" : "monitor_out_packets" }, + { "type" : "longSum", "name" : "intercept_conn_num", "fieldName" : "intercept_conn_num" }, + { "type" : "longSum", "name" : "intercept_in_bytes", "fieldName" : "intercept_in_bytes" }, + { "type" : "longSum", "name" : "intercept_out_bytes", "fieldName" : "intercept_out_bytes" }, + { "type" : "longSum", "name" : "intercept_in_packets", "fieldName" : "intercept_in_packets" }, + { "type" : "longSum", "name" : "intercept_out_packets", "fieldName" : "intercept_out_packets" }, + { "type" : "longSum", "name" : "pinning_num", "fieldName" : "pinning_num" }, + { "type" : "longSum", "name" : "not_pinning_num", "fieldName" : "not_pinning_num" }, + { "type" : "longSum", "name" : "maybe_pinning_num", "fieldName" : "maybe_pinning_num" }, + { "type" : "longSum", "name" : "alert_bytes", "fieldName" : "alert_bytes" }, + { "type" : "longSum", "name" : "ins_bytes", "fieldName" : "insert_bytes" }, + { "type" : "longSum", "name" : "hijk_bytes", "fieldName" : "hijack_bytes" }, + { "type" : "longSum", "name" : "ad_reflection_bytes", "fieldName" : "ad_reflection_bytes" }, + { "type" : "longSum", "name" : "ad_flood_bytes", "fieldName" : "ad_flood_bytes" }, + { "type" : "longSum", "name" : "ad_cc_bytes", "fieldName" : "ad_cc_bytes" }, + { "type" : "longSum", "name" : "intcp_mon_num", "fieldName" : "intercept_monitor_conn_num" }, + { "type" : "longSum", "name" : "intcp_allow_num", "fieldName" : "intercept_allow_conn_num" }, + { "type" : "longSum", "name" : "intcp_deny_num", "fieldName" : "intercept_deny_conn_num" }, + { "type" : "longSum", "name" : "intcp_rdirt_num", "fieldName" : "intercept_redirect_conn_num" }, + { "type" : "longSum", "name" : "intcp_repl_num", "fieldName" : "intercept_replace_conn_num" }, + { "type" : "longSum", "name" : "intcp_hijk_num", "fieldName" : "intercept_hijack_conn_num" }, + { "type" : "longSum", "name" : "intcp_ins_num", "fieldName" : "intercept_insert_conn_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT5S", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{} + }, + "tuningConfig": { + "type": "kafka", + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-METRICS-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/traffic_protocol_stat_log.json b/parcels/roles/init/druid/files/druid_topology/tasks/traffic_protocol_stat_log.json new file mode 100644 index 0000000..b9fb348 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/traffic_protocol_stat_log.json @@ -0,0 +1,63 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "traffic_protocol_stat_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "stat_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "protocol_id", + "isp", + "data_center", + { + "name": "entrance_id", + "type": "long" + } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "s2c_byte_num" }, + { "type" : "longSum", "name" : "c2s_ipfrag_num", "fieldName" : "c2s_ipfrag_num" }, + { "type" : "longSum", "name" : "s2c_ipfrag_num", "fieldName" : "s2c_ipfrag_num" }, + { "type" : "longSum", "name" : "c2s_tcp_lostlen", "fieldName" : "c2s_tcp_lostlen" }, + { "type" : "longSum", "name" : "s2c_tcp_lostlen", "fieldName" : "s2c_tcp_lostlen" }, + { "type" : "longSum", "name" : "c2s_tcp_unorder_num", "fieldName" : "c2s_tcp_unorder_num" }, + { "type" : "longSum", "name" : "s2c_tcp_unorder_num", "fieldName" : "s2c_tcp_unorder_num" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT15S", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{} + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 5000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "TRAFFIC-PROTOCOL-STAT-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/tasks/traffic_summary_log.json b/parcels/roles/init/druid/files/druid_topology/tasks/traffic_summary_log.json new file mode 100644 index 0000000..9235705 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/tasks/traffic_summary_log.json @@ -0,0 +1,72 @@ +{ + "type": "kafka", + "dataSchema": { + "dataSource": "traffic_summary_log", + "parser": { + "type": "string", + "parseSpec": { + "format": "json", + "timestampSpec": { + "column": "common_recv_time", + "format": "posix" + }, + "dimensionsSpec": { + "dimensions": [ + "isp", + "data_center", + "schema_type", + { "name": "entrance_id", "type": "long" } + ] + } + } + }, + "metricsSpec" : [ + { "type" : "HLLSketchBuild", "name" : "ip_object", "fieldName" : "common_client_ip" }, + { "type" : "longSum", "name" : "sessions", "fieldName" : "common_sessions" }, + { "type" : "longSum", "name" : "c2s_pkt_num", "fieldName" : "common_c2s_pkt_num" }, + { "type" : "longSum", "name" : "s2c_pkt_num", "fieldName" : "common_s2c_pkt_num" }, + { "type" : "longSum", "name" : "c2s_byte_num", "fieldName" : "common_c2s_byte_num" }, + { "type" : "longSum", "name" : "s2c_byte_num", "fieldName" : "common_s2c_byte_num" }, + { "type" : "longSum", "name" : "one_sided_connections", "fieldName" : "one_sided_connections" }, + { "type" : "longSum", "name" : "uncategorized_bytes", "fieldName" : "uncategorized_bytes" }, + { "type" : "longSum", "name" : "fragmentation_packets", "fieldName" : "fragmentation_packets" }, + { "type" : "longSum", "name" : "sequence_gap_loss", "fieldName" : "sequence_gap_loss" }, + { "type" : "longSum", "name" : "unorder_packets", "fieldName" : "unorder_packets" } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "DAY", + "queryGranularity": {"type": "period", "period": "PT15S", "timeZone": "Asia/Hong_Kong"}, + "rollup": true + }, + "transformSpec" :{ + "transforms":[ + {"type": "expression","name": "isp","expression": "common_isp"}, + {"type": "expression","name": "data_center","expression": "common_data_center"}, + {"type": "expression","name": "entrance_id","expression": "common_entrance_id"}, + {"type": "expression","name": "schema_type","expression": "common_schema_type"}, + {"type": "expression", "name": "one_sided_connections", "expression": "if(common_stream_dir!=3,common_sessions,0)"}, + {"type": "expression", "name": "uncategorized_bytes", "expression": "if(array_ordinal(string_to_array(common_protocol_label,'/'),1)=='UNCATEGORIZED',\"common_c2s_byte_num\"+\"common_s2c_byte_num\",0)"}, + {"type": "expression","name": "fragmentation_packets","expression": "\"common_c2s_ipfrag_num\"+\"common_s2c_ipfrag_num\""}, + {"type": "expression","name": "sequence_gap_loss","expression": "\"common_c2s_tcp_lostlen\"+\"common_s2c_tcp_lostlen\""}, + {"type": "expression","name": "unorder_packets","expression": "\"common_c2s_tcp_unorder_num\"+\"common_s2c_tcp_unorder_num\""} + ] + } + }, + "tuningConfig": { + "type": "kafka", + "maxRowsPerSegment": 20000000, + "reportParseExceptions": false + }, + "ioConfig": { + "topic": "CONNECTION-RECORD-COMPLETED-LOG", + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT1H", + "completionTimeout": "PT30M", + "consumerProperties": { + "bootstrap.servers": "kafkabootstrap", + "group.id": "kafka-indexing-service" + } + } +} diff --git a/parcels/roles/init/druid/files/druid_topology/topology_list.txt b/parcels/roles/init/druid/files/druid_topology/topology_list.txt new file mode 100644 index 0000000..d68b955 --- /dev/null +++ b/parcels/roles/init/druid/files/druid_topology/topology_list.txt @@ -0,0 +1,21 @@ +client_internal_ip_hot_log.json +proxy_event_hits_log.json +security_event_hits_log.json +server_external_ip_hot_log.json +sys_storage_log.json +top_client_ip_log.json +top_external_host_log.json +top_internal_host_log.json +top_server_ip_log.json +top_urls_log.json +top_urls_log_proxy_hot.json +top_urls_log_security_hot.json +top_user_log_hot.json +top_user_log.json +top_website_domain_log_hot.json +top_website_domain_log.json +traffic_app_stat_log.json +traffic_metrics_log.json +traffic_protocol_stat_log.json +traffic_summary_log.json + diff --git a/parcels/roles/init/druid/files/push_segments.sh b/parcels/roles/init/druid/files/push_segments.sh new file mode 100755 index 0000000..a22c82a --- /dev/null +++ b/parcels/roles/init/druid/files/push_segments.sh @@ -0,0 +1,25 @@ +#!/bin/bash +source /etc/profile + +hadoop fs -test -e /druid/segments +if [ $? -ne 0 ] ;then +hadoop fs -mkdir -p /druid/segments +fi + +for dir in `ls ./segments` +do + hadoop fs -test -e /druid/segments/$dir + if [ $? -eq 0 ] ;then + for file in `ls ./segments/$dir` + do + hadoop fs -test -e /druid/segments/$dir/$file + if [ $? -ne 0 ] ;then + hadoop fs -put ./segments/$dir/$file /druid/segments/$dir/ + fi + done + else + hadoop fs -put ./segments/$dir /druid/segments/ + fi + +done + diff --git a/parcels/roles/init/druid/files/segments.zip b/parcels/roles/init/druid/files/segments.zip Binary files differnew file mode 100644 index 0000000..c1e4c71 --- /dev/null +++ b/parcels/roles/init/druid/files/segments.zip diff --git a/parcels/roles/init/druid/files/segments/client_internal_ip_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_39.273Z/0_index.zip b/parcels/roles/init/druid/files/segments/client_internal_ip_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_39.273Z/0_index.zip Binary files differnew file mode 100644 index 0000000..4879b1a --- /dev/null +++ b/parcels/roles/init/druid/files/segments/client_internal_ip_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_39.273Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/proxy_event_hits_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_41.641Z/0_index.zip b/parcels/roles/init/druid/files/segments/proxy_event_hits_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_41.641Z/0_index.zip Binary files differnew file mode 100644 index 0000000..ae5b39a --- /dev/null +++ b/parcels/roles/init/druid/files/segments/proxy_event_hits_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_41.641Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/proxy_event_urls_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_43.849Z/0_index.zip b/parcels/roles/init/druid/files/segments/proxy_event_urls_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_43.849Z/0_index.zip Binary files differnew file mode 100644 index 0000000..aa6b76b --- /dev/null +++ b/parcels/roles/init/druid/files/segments/proxy_event_urls_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_43.849Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/security_event_hits_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_46.392Z/0_index.zip b/parcels/roles/init/druid/files/segments/security_event_hits_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_46.392Z/0_index.zip Binary files differnew file mode 100644 index 0000000..7e3f6ba --- /dev/null +++ b/parcels/roles/init/druid/files/segments/security_event_hits_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_46.392Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/security_event_urls_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_48.458Z/0_index.zip b/parcels/roles/init/druid/files/segments/security_event_urls_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_48.458Z/0_index.zip Binary files differnew file mode 100644 index 0000000..b7f6a02 --- /dev/null +++ b/parcels/roles/init/druid/files/segments/security_event_urls_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_48.458Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/server_external_ip_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_50.092Z/0_index.zip b/parcels/roles/init/druid/files/segments/server_external_ip_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_50.092Z/0_index.zip Binary files differnew file mode 100644 index 0000000..3ca0642 --- /dev/null +++ b/parcels/roles/init/druid/files/segments/server_external_ip_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_50.092Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/sys_storage_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_52.202Z/0_index.zip b/parcels/roles/init/druid/files/segments/sys_storage_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_52.202Z/0_index.zip Binary files differnew file mode 100644 index 0000000..05c9179 --- /dev/null +++ b/parcels/roles/init/druid/files/segments/sys_storage_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_52.202Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/top_client_ip_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_53.675Z/0_index.zip b/parcels/roles/init/druid/files/segments/top_client_ip_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_53.675Z/0_index.zip Binary files differnew file mode 100644 index 0000000..8b5b27a --- /dev/null +++ b/parcels/roles/init/druid/files/segments/top_client_ip_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_53.675Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/top_external_host_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_55.845Z/0_index.zip b/parcels/roles/init/druid/files/segments/top_external_host_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_55.845Z/0_index.zip Binary files differnew file mode 100644 index 0000000..3bc1a01 --- /dev/null +++ b/parcels/roles/init/druid/files/segments/top_external_host_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_55.845Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/top_internal_host_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_58.433Z/0_index.zip b/parcels/roles/init/druid/files/segments/top_internal_host_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_58.433Z/0_index.zip Binary files differnew file mode 100644 index 0000000..996bec3 --- /dev/null +++ b/parcels/roles/init/druid/files/segments/top_internal_host_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_58.433Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/top_server_ip_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_59.654Z/0_index.zip b/parcels/roles/init/druid/files/segments/top_server_ip_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_59.654Z/0_index.zip Binary files differnew file mode 100644 index 0000000..cf4648e --- /dev/null +++ b/parcels/roles/init/druid/files/segments/top_server_ip_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_13_59.654Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/top_urls_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_01.484Z/0_index.zip b/parcels/roles/init/druid/files/segments/top_urls_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_01.484Z/0_index.zip Binary files differnew file mode 100644 index 0000000..b402447 --- /dev/null +++ b/parcels/roles/init/druid/files/segments/top_urls_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_01.484Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/top_user_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_04.063Z/0_index.zip b/parcels/roles/init/druid/files/segments/top_user_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_04.063Z/0_index.zip Binary files differnew file mode 100644 index 0000000..84fb3ed --- /dev/null +++ b/parcels/roles/init/druid/files/segments/top_user_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_04.063Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/top_website_domain_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_06.100Z/0_index.zip b/parcels/roles/init/druid/files/segments/top_website_domain_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_06.100Z/0_index.zip Binary files differnew file mode 100644 index 0000000..9221dd2 --- /dev/null +++ b/parcels/roles/init/druid/files/segments/top_website_domain_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_06.100Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/traffic_app_stat_log/30000101T000000.000Z_30000102T000000.000Z/2021-07-02T02_43_09.615Z/0_index.zip b/parcels/roles/init/druid/files/segments/traffic_app_stat_log/30000101T000000.000Z_30000102T000000.000Z/2021-07-02T02_43_09.615Z/0_index.zip Binary files differnew file mode 100644 index 0000000..63c1bf7 --- /dev/null +++ b/parcels/roles/init/druid/files/segments/traffic_app_stat_log/30000101T000000.000Z_30000102T000000.000Z/2021-07-02T02_43_09.615Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/traffic_metrics_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_08.462Z/0_index.zip b/parcels/roles/init/druid/files/segments/traffic_metrics_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_08.462Z/0_index.zip Binary files differnew file mode 100644 index 0000000..561be3a --- /dev/null +++ b/parcels/roles/init/druid/files/segments/traffic_metrics_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_08.462Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/traffic_protocol_stat_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_09.706Z/0_index.zip b/parcels/roles/init/druid/files/segments/traffic_protocol_stat_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_09.706Z/0_index.zip Binary files differnew file mode 100644 index 0000000..d2917f1 --- /dev/null +++ b/parcels/roles/init/druid/files/segments/traffic_protocol_stat_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_09.706Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/traffic_summary_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_12.193Z/0_index.zip b/parcels/roles/init/druid/files/segments/traffic_summary_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_12.193Z/0_index.zip Binary files differnew file mode 100644 index 0000000..58cbd03 --- /dev/null +++ b/parcels/roles/init/druid/files/segments/traffic_summary_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_12.193Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/user_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_14.350Z/0_index.zip b/parcels/roles/init/druid/files/segments/user_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_14.350Z/0_index.zip Binary files differnew file mode 100644 index 0000000..67b1b2e --- /dev/null +++ b/parcels/roles/init/druid/files/segments/user_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_14.350Z/0_index.zip diff --git a/parcels/roles/init/druid/files/segments/website_domain_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_16.425Z/0_index.zip b/parcels/roles/init/druid/files/segments/website_domain_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_16.425Z/0_index.zip Binary files differnew file mode 100644 index 0000000..8c7e660 --- /dev/null +++ b/parcels/roles/init/druid/files/segments/website_domain_hot_log/30000101T000000.000Z_30000102T000000.000Z/2021-04-02T03_14_16.425Z/0_index.zip diff --git a/parcels/roles/init/druid/tasks/main.yml b/parcels/roles/init/druid/tasks/main.yml new file mode 100644 index 0000000..7d2dbb5 --- /dev/null +++ b/parcels/roles/init/druid/tasks/main.yml @@ -0,0 +1,85 @@ +- name: hadoop_ip to ansible variable + set_fact: hadoop_ip="{{groups.hadoop[0]}}" + +- name: copy segments to hadoop + copy: + src: '../files/segments.zip' + dest: '{{ install_path }}/segments.zip' + force: true + run_once: true + delegate_to: '{{ hadoop_ip }}' + +- name: copy segments to hadoop + copy: + src: '../files/push_segments.sh' + dest: '{{ install_path }}/push_segments.sh' + force: true + mode: 0755 + run_once: true + delegate_to: '{{ hadoop_ip }}' + +- name: create druid dir and put segments to hdfs + shell: source /etc/profile && hadoop fs -mkdir -p /druid/segments + register: nums_out + run_once: true + delegate_to: '{{ hadoop_ip }}' + ignore_errors: yes + +- name: unpack segments.zip + unarchive: + src: 'segments.zip' + dest: '{{ install_path }}/' + run_once: true + delegate_to: '{{ hadoop_ip }}' + +- name: create druid dir and put segments to hdfs + shell: source /etc/profile && cd {{ install_path }}/ && sh push_segments.sh + register: nums_out + run_once: true + delegate_to: '{{ hadoop_ip }}' + +- name: check if {{ mariadb_druid_database }} exist + shell: mysql -s -uroot -p{{ galaxy_mariadb_pin }} -P3306 -h{{ gateway_keepalive_host }} -e "SELECT COUNT(1) FROM druid.druid_segments WHERE start = '3000-01-02T00:00:00.000Z'" + register: segments_nums + run_once: true + delegate_to: "127.0.0.1" + +- name: copy segments.sql to ~ + copy: + src: '../files/druid_segments-tsg3.0.sql' + dest: '~/druid_segments-tsg3.0.sql' + force: true + run_once: true + delegate_to: '127.0.0.1' + when: segments_nums.stdout != '20' + +- name: insert segments to mariadb + shell: 'mysql -uroot -p{{ galaxy_mariadb_pin }} -P3306 -h{{ gateway_keepalive_host }} {{ mariadb_druid_database }} < ~/druid_segments-tsg3.0.sql' + run_once: true + delegate_to: '127.0.0.1' + when: segments_nums.stdout != '20' + +- name: copy druid_topology to {{ install_path }} + copy: + src: '../files/druid_topology' + dest: '{{ install_path }}/' + force: true + +- name: + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + backup: yes + with_items: + - { src: 'ingestionTask.sh.j2', dest: '{{ install_path }}/druid_topology/ingestionTask.sh' } + - { src: 'post_rule.sh.j2', dest: '{{ install_path }}/druid_topology/rule/post_rule.sh' } + +- name: commit druid topology + shell: cd {{ install_path }}/druid_topology && chmod +x ingestionTask.sh && sh ingestionTask.sh + run_once: true + delegate_to: '{{ groups.druid[0] }}' + +- name: commit druid hot default deletion strategy + shell: cd {{ install_path }}/druid_topology/rule && chmod +x post_rule.sh && sh post_rule.sh + run_once: true + delegate_to: '{{ groups.druid[0] }}' diff --git a/parcels/roles/init/druid/templates/ingestionTask.sh.j2 b/parcels/roles/init/druid/templates/ingestionTask.sh.j2 new file mode 100644 index 0000000..6fa0866 --- /dev/null +++ b/parcels/roles/init/druid/templates/ingestionTask.sh.j2 @@ -0,0 +1,25 @@ +#!/bin/bash + +#note:该脚本应该和任务列表文件tasks.txt,*.json文件存放于同一目录下 +source /etc/profile +task_name=`cat topology_list.txt` +kafka_host={{ consumer_kafka_servers }} +druid_host={{ groups.druid[0] }} + +curl -i -XGET 'http://'$druid_host':8081/druid/indexer/v1/leader' +num=$? +while [ $num -gt "0" ];do + sleep 5 + curl -s -XGET 'http://'$druid_host':8081/druid/indexer/v1/leader' + num=$? +done +cp ./tasks/* ./ + +for var in ${task_name[@]}; +do +sed -i 's/kafkabootstrap/'$kafka_host'/' ${var} +curl -X 'POST' -H 'Content-Type:application/json' -d @${var} http://$druid_host:8081/druid/indexer/v1/supervisor +echo "'${var}' 任务启动成功" +sleep 2 +done + diff --git a/parcels/roles/init/druid/templates/post_rule.sh.j2 b/parcels/roles/init/druid/templates/post_rule.sh.j2 new file mode 100755 index 0000000..f97c1b4 --- /dev/null +++ b/parcels/roles/init/druid/templates/post_rule.sh.j2 @@ -0,0 +1,28 @@ +#!/bin/bash + +base_dir=`dirname $0` +echo $base_dir + +#router节点IP +router_ip={{ groups.druid[0] }} + +common_task_name=`cat $base_dir/common_task.txt` +common_rule_file="$base_dir/common_data_rule.json" + +hot_task_name=`cat $base_dir/hot_task.txt` +hot_rule_file="$base_dir/hot_data_rule.json" + +for var in ${common_task_name[@]}; +do +curl -X 'POST' -H 'Content-Type:application/json' -d @$common_rule_file http://${router_ip}:8089/druid/coordinator/v1/rules/${var} +echo "'${var}' 任务启动成功" +sleep 2 +done + +for var in ${hot_task_name[@]}; +do +curl -X 'POST' -H 'Content-Type:application/json' -d @$hot_rule_file http://${router_ip}:8089/druid/coordinator/v1/rules/${var} +echo "'${var}' 任务启动成功" +sleep 2 +done + diff --git a/parcels/roles/init/galaxy-gateway-keepalive/tasks/main.yml b/parcels/roles/init/galaxy-gateway-keepalive/tasks/main.yml new file mode 100644 index 0000000..6bfa4fd --- /dev/null +++ b/parcels/roles/init/galaxy-gateway-keepalive/tasks/main.yml @@ -0,0 +1,27 @@ +- name: copy kp_daemon.sh + template: src=kp_daemon.sh.j2 dest=/etc/keepalived/kp_daemon.sh force=true mode=u+x + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + +- name: copy check_gateway.sh + template: src=check_gateway.sh.j2 dest=/etc/keepalived/check_gateway.sh force=true mode=u+x + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + + +- name: start kp_daemon.sh + shell: source /etc/profile && nohup sh /etc/keepalived/kp_daemon.sh > /dev/null 2>&1 & + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } diff --git a/parcels/roles/init/galaxy-gateway-keepalive/templates/check_gateway.sh.j2 b/parcels/roles/init/galaxy-gateway-keepalive/templates/check_gateway.sh.j2 new file mode 100644 index 0000000..b8755e0 --- /dev/null +++ b/parcels/roles/init/galaxy-gateway-keepalive/templates/check_gateway.sh.j2 @@ -0,0 +1,84 @@ +#!/bin/bash + +function checkQgw() { +QGW_COUNT=$(ps -ef | grep galaxy-qgw-service | grep -v grep | wc -l) +#判断是否都挂掉了 +if [ $QGW_COUNT -eq 0 ] +then + #如果挂掉了，就启动 + cd {{ install_path }}/{{ soft_home_path }}/galaxy-qgw-service && docker-compose down + cd {{ install_path }}/{{ soft_home_path }}/galaxy-qgw-service && docker-compose up -d + #等5秒钟后，再次查看是否 启动成功 + sleep 5 + #如果没有启动起来，就直接干掉keepalived + QGW_COUNT=$(ps -ef | grep galaxy-qgw-service | grep -v grep | wc -l) + if [ $QGW_COUNT -eq 0 ] + then + echo "干掉keepalived" + killall keepalived + fi +fi +} + +function checkChproxy() { +CHPROXY_COUNT=$(ps -ef | grep chproxy | grep -v grep | wc -l) +#判断是否都挂掉了 +if [ $CHPROXY_COUNT -eq 0 ] +then + #如果挂掉了，就启动 + cd {{ install_path }}/{{ soft_home_path }}/galaxy-chproxy && docker-compose down + cd {{ install_path }}/{{ soft_home_path }}/galaxy-chproxy && docker-compose up -d + #等5秒钟后，再次查看是否 启动成功 + sleep 5 + #如果没有启动起来，就直接干掉keepalived + CHPROXY_COUNT=$(ps -ef | grep chproxy | grep -v grep | wc -l) + if [ $CHPROXY_COUNT -eq 0 ] + then + echo "干掉keepalived" + killall keepalived + fi +fi +} + +function checkNginx() { +NGINX_COUNT=$(ps -ef | grep nginx | grep -v grep | wc -l) +#判断是否都挂掉了 +if [ $NGINX_COUNT -eq 0 ] +then + #如果挂掉了，就启动 + cd {{ install_path }}/{{ soft_home_path }}/galaxy-app-nginx && docker-compose down + cd {{ install_path }}/{{ soft_home_path }}/galaxy-app-nginx && docker-compose up -d + #等5秒钟后，再次查看是否 启动成功 + sleep 5 + #如果没有启动起来，就直接干掉keepalived + NGINX_COUNT=$(ps -ef | grep nginx | grep -v grep | wc -l) + if [ $NGINX_COUNT -eq 0 ] + then + echo "干掉keepalived" + killall keepalived + fi +fi +} + +function checkMariadb() { +MARIADB_COUNT=$(ps -ef | grep mysql | grep -v grep | wc -l) +#判断是否都挂掉了 +if [ $MARIADB_COUNT -eq 0 ] +then + #如果挂掉了，就启动 + service mysql restart + #等5秒钟后，再次查看是否 启动成功 + sleep 5 + #如果没有启动起来，就直接干掉keepalived + MARIADB_COUNT=$(ps -ef | grep mysql | grep -v grep | wc -l) + if [ $MARIADB_COUNT -eq 0 ] + then + echo "干掉keepalived" + killall keepalived + fi +fi +} + +checkChproxy +checkNginx +checkMariadb diff --git a/parcels/roles/init/galaxy-gateway-keepalive/templates/kp_daemon.sh.j2 b/parcels/roles/init/galaxy-gateway-keepalive/templates/kp_daemon.sh.j2 new file mode 100644 index 0000000..416ac7e --- /dev/null +++ b/parcels/roles/init/galaxy-gateway-keepalive/templates/kp_daemon.sh.j2 @@ -0,0 +1,25 @@ +#!/bin/bash + +keepalivedPath=/etc/keepalived +pidFile=/var/run/keepalived.pid +logPath=$keepalivedPath/keepalived_daemon.log + +while true;do + CHPROXY_COUNT=$(ps -ef | grep chproxy | grep -v grep | wc -l) + NGINX_COUNT=$(ps -ef | grep nginx | grep -v grep | wc -l) + MARIADB_COUNT=$(ps -ef | grep mysql | grep -v grep | wc -l) + + if [[ $MARIADB_COUNT -eq "0" || $CHPROXY_COUNT -eq "0" || $NGINX_COUNT -eq "0" ]] + then + echo "时间：`date '+%Y-%m-%d %T'` APP服务进程不存在,关闭keepalived进程" >> $logPath + systemctl stop keepalived + elif [ ! -f $pidFile ]; then + echo "时间：`date '+%Y-%m-%d %T'` keepalived进程不存在,如果服务都健康,开始启动keepalived" >> $logPath + if [[ $MARIADB_COUNT -gt "0" && $CHPROXY_COUNT -gt "0" && $NGINX_COUNT -gt "0" ]] #haproxy和nginx都不存在停止keepalived + then + systemctl start keepalived + fi + fi + sleep 5 +done + diff --git a/parcels/roles/init/galaxy-gateway-keepalive/vars/main.yml b/parcels/roles/init/galaxy-gateway-keepalive/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/init/galaxy-gateway-keepalive/vars/main.yml diff --git a/parcels/roles/init/galaxy-hos-service/tasks/main.yml b/parcels/roles/init/galaxy-hos-service/tasks/main.yml new file mode 100644 index 0000000..f32a622 --- /dev/null +++ b/parcels/roles/init/galaxy-hos-service/tasks/main.yml @@ -0,0 +1,120 @@ +- name: master_ip to ansible variable + set_fact: master_ip="{{groups.hos[0]}}" + +- name: wait for hos start + wait_for: + host: '{{ master_ip }}' + port: 8186 + timeout: 60 + +- name: init hos service + shell: "curl --location -s -w %{http_code} --request PUT 'http://{{ master_ip }}:8186/admin/initialize' --header 'token: c21f969b5f03d33d43e04f8f136e7682'" + register: initialize + run_once: true + delegate_facts: true + delegate_to: '{{ master_ip }}' + +- name: check hos init status + fail: + msg: "HOS 内部初始化接口执行失败!请检查" + when: initialize.stdout != '200' + +- name: delete firewall_hos_bucket + shell: "curl --location -s -w %{http_code} --request DELETE 'http://{{ master_ip }}:8186/hos/firewall_hos_bucket' --header 'token: c21f969b5f03d33d43e04f8f136e7682'" + register: delete + run_once: true + delegate_facts: true + delegate_to: '{{ master_ip }}' + +- name: check delete firewall_hos_bucket status + fail: + msg: "HOS 删除 firewall_hos_bucket 桶异常" + when: delete.stdout != '204' + + +- name: delete proxy_hos_bucket + shell: "curl --location -s -w %{http_code} --request DELETE 'http://{{ master_ip }}:8186/hos/proxy_hos_bucket' --header 'token: c21f969b5f03d33d43e04f8f136e7682'" + register: delete + run_once: true + delegate_facts: true + delegate_to: '{{ master_ip }}' + +- name: check delete proxy_hos_bucket status + fail: + msg: "HOS 删除 proxy_hos_bucket 桶异常" + when: delete.stdout != '204' + + +- name: delete session_record_hos_bucket + shell: "curl --location -s -w %{http_code} --request DELETE 'http://{{ master_ip }}:8186/hos/session_record_hos_bucket' --header 'token: c21f969b5f03d33d43e04f8f136e7682'" + register: delete + run_once: true + delegate_facts: true + delegate_to: '{{ master_ip }}' + +- name: check delete session_record_hos_bucket status + fail: + msg: "HOS 删除 session_record_hos_bucket 桶异常" + when: delete.stdout != '204' + + +- name: create firewall_hos_bucket + shell: "curl --location -s -w %{http_code} --request PUT 'http://{{ master_ip }}:8186/hos/firewall_hos_bucket' --header 'token: c21f969b5f03d33d43e04f8f136e7682'" + register: table1 + run_once: true + delegate_facts: true + delegate_to: '{{ master_ip }}' + +- name: check create firewall_hos_bucket status + fail: + msg: "HOS初始化 firewall_hos_bucket 桶失败!请检查" + when: table1.stdout.find('200') == '-1' and table1.stdout.find('409') == '-1' + +- name: create proxy_hos_bucket + shell: "curl --location -s -w %{http_code} --request PUT 'http://{{ master_ip }}:8186/hos/proxy_hos_bucket' --header 'token: c21f969b5f03d33d43e04f8f136e7682'" + register: table2 + run_once: true + delegate_facts: true + delegate_to: '{{ master_ip }}' + +- name: check create proxy_hos_bucket status + fail: + msg: "HOS初始化 proxy_hos_bucket 桶失败!请检查" + when: table2.stdout.find('200') == '-1' and table2.stdout.find('409') == '-1' + +- name: create session_record_hos_bucket + shell: "curl --location -s -w %{http_code} --request PUT 'http://{{ master_ip }}:8186/hos/session_record_hos_bucket' --header 'token: c21f969b5f03d33d43e04f8f136e7682'" + register: table3 + run_once: true + delegate_facts: true + delegate_to: '{{ master_ip }}' + +- name: check create session_record_hos_bucket status + fail: + msg: "HOS初始化 session_record_hos_bucket 桶失败!请检查" + when: table3.stdout.find('200') == '-1' and table3.stdout.find('409') == '-1' + +- name: Does proxy_hos_bucket exist + shell: "curl --location -s -w %{http_code} --request GET 'http://{{ master_ip }}:8186/hos/' --header 'token: c21f969b5f03d33d43e04f8f136e7682' |grep proxy_hos_bucket | wc -l" + register: bucketa_out + +- fail: + msg: "proxy_hos_bucket 未创建成功 请重新执行" + when: bucketa_out.stdout != '1' + +- name: Does session_record_hos_bucket exist + shell: "curl --location -s -w %{http_code} --request GET 'http://{{ master_ip }}:8186/hos/' --header 'token: c21f969b5f03d33d43e04f8f136e7682' |grep session_record_hos_bucket | wc -l" + register: bucketb_out + +- fail: + msg: "session_record_hos_bucket 未创建成功 请重新执行" + when: bucketb_out.stdout != '1' + +- name: Does firewall_hos_bucket exist + shell: "curl --location -s -w %{http_code} --request GET 'http://{{ master_ip }}:8186/hos/' --header 'token: c21f969b5f03d33d43e04f8f136e7682' |grep firewall_hos_bucket | wc -l" + register: bucketc_out + +- fail: + msg: "firewall_hos_bucket 未创建成功 请重新执行" + when: bucketc_out.stdout != '1' + diff --git a/parcels/roles/init/galaxy-job-service/files/mysql b/parcels/roles/init/galaxy-job-service/files/mysql Binary files differnew file mode 100755 index 0000000..66af1a1 --- /dev/null +++ b/parcels/roles/init/galaxy-job-service/files/mysql diff --git a/parcels/roles/init/galaxy-job-service/tasks/main.yml b/parcels/roles/init/galaxy-job-service/tasks/main.yml new file mode 100644 index 0000000..d0bcffa --- /dev/null +++ b/parcels/roles/init/galaxy-job-service/tasks/main.yml @@ -0,0 +1,64 @@ +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181" + when: '(groups.zookeeper|length) == 3' + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181,{{groups.zookeeper[3]}}:2181,{{groups.zookeeper[4]}}:2181" + when: '(groups.zookeeper|length) >= 5' + +- name: get job_admin_servers to ansible variable + set_fact: job_admin_servers="http://{{ groups.services[0] }}:8184/xxl-job-admin,http://{{ groups.services[1] }}:8184/xxl-job-admin" + when: '(groups.services|length) == 2' + +- name: get job_admin_servers to ansible variable + set_fact: job_admin_servers="http://{{ groups.services[0] }}:8184/xxl-job-admin,http://{{ groups.services[1] }}:8184/xxl-job-admin,http://{{ groups.services[2] }}:8184/xxl-job-admin" + when: '(groups.services|length) == 3' + +- name: get job_executor_servers to ansible variable + set_fact: job_executor_servers="http://{{ groups.services[0] }}:8886,http://{{ groups.services[1] }}:8886" + when: '(groups.services|length) == 2' + +- name: get job_executor_servers to ansible variable + set_fact: job_executor_servers="http://{{ groups.services[0] }}:8886,http://{{ groups.services[1] }}:8886,http://{{ groups.services[2] }}:8886" + when: '(groups.services|length) == 3' + +- name: copy mysql + copy: + src: '../files/mysql' + dest: '/usr/bin/' + force: true + backup: yes + run_once: true + delegate_to: '{{groups.services[0]}}' + +- name: change mode mysql + shell: chmod +x /usr/bin/mysql + run_once: true + delegate_to: '{{groups.services[0]}}' + +- name: copy {{ admin_image_name }} xxl_job.sql.j2 + template: + src: xxl_job.sql.j2 + dest: '~/xxl_job.sql' + backup: yes + run_once: true + delegate_to: '{{groups.services[0]}}' + +- name: copy {{ admin_image_name }} create_mariadb_tables.sh.j2 + template: + src: create_mariadb_tables.sh.j2 + dest: '~/create_mariadb_tables.sh' + mode: 0755 + backup: yes + run_once: true + delegate_to: '{{groups.services[0]}}' + +- name: create the mysql's tables and insert some datas + shell: cd ~ && ./create_mariadb_tables.sh + run_once: true + delegate_to: '{{groups.services[0]}}' + +- name: start {{ job_image_container_name }} container + docker_compose: + project_src: '{{ install_path }}/{{ job_soft_home_path }}' + diff --git a/parcels/roles/init/galaxy-job-service/templates/create_mariadb_tables.sh.j2 b/parcels/roles/init/galaxy-job-service/templates/create_mariadb_tables.sh.j2 new file mode 100644 index 0000000..c072c5d --- /dev/null +++ b/parcels/roles/init/galaxy-job-service/templates/create_mariadb_tables.sh.j2 @@ -0,0 +1,6 @@ +#!/bin/bash + +mysql -uroot -p{{ galaxy_mariadb_pin }} -P3306 -h{{ gateway_keepalive_host }} -e "create database if not exists {{ mariadb_job_database }} default character set utf8mb4 collate utf8mb4_general_ci;" + + +mysql -uroot -p{{ galaxy_mariadb_pin }} -P3306 -h{{ gateway_keepalive_host }} {{ mariadb_job_database }} < xxl_job.sql diff --git a/parcels/roles/init/galaxy-job-service/templates/xxl_job.sql.j2 b/parcels/roles/init/galaxy-job-service/templates/xxl_job.sql.j2 new file mode 100644 index 0000000..257efaf --- /dev/null +++ b/parcels/roles/init/galaxy-job-service/templates/xxl_job.sql.j2 @@ -0,0 +1,193 @@ +/* +Navicat MySQL Data Transfer + +Source Server : 192.168.44.3 +Source Server Version : 50505 +Source Host : 192.168.44.3:3306 +Source Database : xxl_job + +Target Server Type : MYSQL +Target Server Version : 50505 +File Encoding : 65001 + +Date: 2020-10-19 10:28:42 +*/ + +SET FOREIGN_KEY_CHECKS=0; + +-- ---------------------------- +-- Table structure for `xxl_job_group` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_group`; +CREATE TABLE `xxl_job_group` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `app_name` varchar(64) NOT NULL COMMENT '执行器AppName', + `title` varchar(12) NOT NULL COMMENT '执行器名称', + `address_type` tinyint(4) NOT NULL DEFAULT 0 COMMENT '执行器地址类型：0=自动注册、1=手动录入', + `address_list` varchar(512) DEFAULT NULL COMMENT '执行器地址列表，多地址逗号分隔', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- +-- Records of xxl_job_group +-- ---------------------------- +INSERT INTO `xxl_job_group` VALUES ('3', 'galaxy-executor', '数据平台业务执行器', '1', '{{ job_executor_servers }}'); + +-- ---------------------------- +-- Table structure for `xxl_job_info` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_info`; +CREATE TABLE `xxl_job_info` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `job_group` int(11) NOT NULL COMMENT '执行器主键ID', + `job_cron` varchar(128) NOT NULL COMMENT '任务执行CRON', + `job_desc` varchar(255) NOT NULL, + `add_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `author` varchar(64) DEFAULT NULL COMMENT '作者', + `alarm_email` varchar(255) DEFAULT NULL COMMENT '报警邮件', + `executor_route_strategy` varchar(50) DEFAULT NULL COMMENT '执行器路由策略', + `executor_handler` varchar(255) DEFAULT NULL COMMENT '执行器任务handler', + `executor_param` varchar(4000) DEFAULT '' COMMENT '执行器任务参数', + `executor_block_strategy` varchar(50) DEFAULT NULL COMMENT '阻塞处理策略', + `executor_timeout` int(11) NOT NULL DEFAULT 0 COMMENT '任务执行超时时间，单位秒', + `executor_fail_retry_count` int(11) NOT NULL DEFAULT 0 COMMENT '失败重试次数', + `glue_type` varchar(50) NOT NULL COMMENT 'GLUE类型', + `glue_source` mediumtext DEFAULT NULL COMMENT 'GLUE源代码', + `glue_remark` varchar(128) DEFAULT NULL COMMENT 'GLUE备注', + `glue_updatetime` datetime DEFAULT NULL COMMENT 'GLUE更新时间', + `child_jobid` varchar(255) DEFAULT NULL COMMENT '子任务ID，多个逗号分隔', + `trigger_status` tinyint(4) NOT NULL DEFAULT 0 COMMENT '调度状态：0-停止，1-运行', + `trigger_last_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '上次调度时间', + `trigger_next_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '下次调度时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=77 DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- +-- Records of xxl_job_info +-- ---------------------------- +INSERT INTO `xxl_job_info` VALUES ('29', '3', '0 0 4 * * ?', '获取CK存储配额-sh(实时压缩负值问题）', '2020-03-26 10:58:09', '2020-08-03 10:04:29', 'zhq', '', 'FAILOVER', 'getStorageQuotaJobHandler', '{\"topic\":\"SYS-STORAGE-LOG\",\"ckDayGrowth\":\"true\"}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-03-26 10:58:09', '', '1', '1603051200000', '1603137600000'); +INSERT INTO `xxl_job_info` VALUES ('36', '3', '0 0/5 * * * ?', '获取所有存储配额-sh', '2020-04-07 21:41:20', '2020-08-03 10:30:05', 'zhq', '', 'FAILOVER', 'getStorageQuotaJobHandler', '{\"topic\":\"SYS-STORAGE-LOG\",\"ckDayGrowth\":\"false\"}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-04-07 21:41:20', '', '1', '1603074600000', '1603074900000'); +INSERT INTO `xxl_job_info` VALUES ('44', '3', '0 0/5 * * * ?', 'Event-Security-Proxy-Top-URLS-By-Hits', '2020-04-26 11:01:44', '2020-10-19 10:20:14', 'galaxy', '', 'FAILOVER', 'httpToKafkaJobHandler', '{\r\n \"url\": \"http://{{ gateway_keepalive_host }}:8089/druid/v2/sql\",\r\n \"method\": \"post\",\r\n \"requestBody\": {\r\n \"query\": \"(SELECT url ,sum( session_num ) AS session_num,TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time FROM security_event_urls_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP, \'PT5M\') GROUP BY url ORDER BY session_num DESC LIMIT 1000) UNION ALL(SELECT url, sum(session_num) AS session_num, TIME_FLOOR(CURRENT_TIMESTAMP, \'PT5M\') as stat_time FROM proxy_event_urls_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP, \'PT5M\'), \'PT5M\', -1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP, \'PT5M\') GROUP BY url ORDER BY session_num DESC LIMIT 1000)\",\r\n \"context\": {\r\n \"skipEmptyBuckets\": \"false\"\r\n },\r\n \"resultFormat\": \"object\"\r\n },\r\n \"resultKey\": \"\",\r\n \"topic\": \"TOP-URLS-LOG\"\r\n}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-04-26 11:01:44', '', '1', '1603074600000', '1603074900000'); +INSERT INTO `xxl_job_info` VALUES ('45', '3', '0 0/5 * * * ?', 'Traffic-Top-Website-Domain', '2020-04-26 11:18:26', '2021-01-02 15:33:03', 'galaxy', '', 'FAILOVER', 'httpToKafkaJobHandler', '{\r\n \"url\": \"http://{{ gateway_keepalive_host }}:8089/druid/v2/sql\",\r\n \"method\": \"post\",\r\n \"requestBody\": {\r\n \"query\": \"(SELECT domain,sum( session_num ) AS session_num,sum( c2s_byte_num ) AS c2s_byte_num,sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num,TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'packets\' as order_by FROM website_domain_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND domain != \'\' GROUP BY domain ORDER BY c2s_pkt_num+s2c_pkt_num DESC LIMIT 1000) UNION ALL (SELECT domain,sum( session_num ) AS session_num,sum( c2s_byte_num ) AS c2s_byte_num,sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num,TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'bytes\' as order_by FROM website_domain_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND domain != \'\' GROUP BY domain ORDER BY c2s_byte_num+s2c_byte_num DESC LIMIT 1000) UNION ALL (SELECT domain,sum(session_num) AS session_num, sum(c2s_byte_num) AS c2s_byte_num, sum(s2c_byte_num) AS s2c_byte_num, sum(c2s_pkt_num) AS c2s_pkt_num,sum(s2c_pkt_num) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'sessions\' as order_by FROM website_domain_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND domain != \'\' GROUP BY domain ORDER BY session_num DESC LIMIT 1000)\",\r\n \"context\": {\r\n \"skipEmptyBuckets\": \"false\"\r\n },\r\n \"resultFormat\": \"object\"\r\n },\r\n \"resultKey\": \"\",\r\n \"topic\": \"TOP-WEBSITE-DOMAIN-LOG\"\r\n}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-04-26 11:18:26', '', '1', '1609739400000', '1609739700000'); +INSERT INTO `xxl_job_info` VALUES ('46', '3', '0 0/5 * * * ?', 'Traffic-Top-User', '2020-04-26 11:23:08', '2021-01-02 15:32:34', 'galaxy', '', 'FAILOVER', 'httpToKafkaJobHandler', '{\r\n \"url\": \"http://{{ gateway_keepalive_host }}:8089/druid/v2/sql\",\r\n \"method\": \"post\",\r\n \"requestBody\": {\r\n \"query\": \"(SELECT common_subscriber_id as subscriber_id, sum( session_num ) AS session_num,sum( c2s_byte_num ) AS c2s_byte_num,sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num, sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'packets\' as order_by FROM user_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_subscriber_id != \'\' GROUP BY common_subscriber_id ORDER BY c2s_pkt_num+s2c_pkt_num DESC LIMIT 1000) UNION ALL (SELECT common_subscriber_id as subscriber_id,sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num, sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'bytes\' as order_by FROM user_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_subscriber_id != \'\' GROUP BY common_subscriber_id ORDER BY c2s_byte_num+s2c_byte_num DESC LIMIT 1000) UNION ALL (SELECT common_subscriber_id as subscriber_id,sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num, sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num,TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'sessions\' as order_by FROM user_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_subscriber_id != \'\' GROUP BY common_subscriber_id ORDER BY session_num DESC LIMIT 1000)\",\r\n \"context\": {\r\n \"skipEmptyBuckets\": \"false\"\r\n },\r\n \"resultFormat\": \"object\"\r\n },\r\n \"resultKey\": \"\",\r\n \"topic\": \"TOP-USER-LOG\"\r\n}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-04-26 11:23:08', '', '1', '1609739400000', '1609739700000'); +INSERT INTO `xxl_job_info` VALUES ('47', '3', '0 0/5 * * * ?', 'Traffic-Top-External-Host', '2020-04-26 11:32:41', '2021-01-02 15:32:02', 'galaxy', '', 'FAILOVER', 'httpToKafkaJobHandler', '{\r\n \"url\": \"http://{{ gateway_keepalive_host }}:8089/druid/v2/sql\",\r\n \"method\": \"post\",\r\n \"requestBody\": {\r\n \"query\": \"(SELECT common_external_ip AS destination, sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num,TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'packets\' as order_by FROM server_external_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_external_ip != \'\' GROUP BY common_external_ip ORDER BY c2s_pkt_num+s2c_pkt_num DESC LIMIT 1000) UNION ALL (SELECT common_external_ip AS destination, sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'bytes\' as order_by FROM server_external_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_external_ip != \'\' GROUP BY common_external_ip ORDER BY c2s_byte_num+s2c_byte_num DESC LIMIT 1000) UNION ALL (SELECT common_external_ip AS destination, sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num, sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'sessions\' as order_by FROM server_external_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_external_ip != \'\' GROUP BY common_external_ip ORDER BY session_num DESC LIMIT 1000)\",\r\n \"context\": {\r\n \"skipEmptyBuckets\": \"false\"\r\n },\r\n \"resultFormat\": \"object\"\r\n },\r\n \"resultKey\": \"\",\r\n \"topic\": \"TOP-EXTERNAL-HOST-LOG\"\r\n}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-04-26 11:32:41', '', '1', '1609739400000', '1609739700000'); +INSERT INTO `xxl_job_info` VALUES ('48', '3', '0 0/5 * * * ?', 'Traffic-Top-Internal-Host', '2020-04-26 11:38:57', '2021-01-02 15:31:13', 'galaxy', '', 'FAILOVER', 'httpToKafkaJobHandler', '{\r\n \"url\": \"http://{{ gateway_keepalive_host }}:8089/druid/v2/sql\",\r\n \"method\": \"post\",\r\n \"requestBody\": {\r\n \"query\": \"(SELECT common_internal_ip AS source, sum( session_num ) AS session_num,sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num, sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'packets\' as order_by FROM client_internal_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_internal_ip != \'\' GROUP BY common_internal_ip ORDER BY c2s_pkt_num+s2c_pkt_num DESC LIMIT 1000) UNION ALL (SELECT common_internal_ip AS source, sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num, sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num,TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'bytes\' as order_by FROM client_internal_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_internal_ip != \'\' GROUP BY common_internal_ip ORDER BY c2s_byte_num+s2c_byte_num DESC LIMIT 1000) UNION ALL (SELECT common_internal_ip AS source, sum( session_num ) AS session_num,sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num, sum( c2s_pkt_num ) AS c2s_pkt_num, sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time, \'sessions\' as order_by FROM client_internal_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_internal_ip != \'\' GROUP BY common_internal_ip ORDER BY session_num DESC LIMIT 1000)\",\r\n \"context\": {\r\n \"skipEmptyBuckets\": \"false\"\r\n },\r\n \"resultFormat\": \"object\"\r\n },\r\n \"resultKey\": \"\",\r\n \"topic\": \"TOP-INTERNAL-HOST-LOG\"\r\n}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-04-26 11:38:57', '', '1', '1609739400000', '1609739700000'); +INSERT INTO `xxl_job_info` VALUES ('58', '3', '0 0 0 1/1 * ? ', '设置统计数据存储配额( Druid 保留天)', '2020-07-30 10:34:34', '2020-10-15 18:27:29', 'zhq', '', 'FAILOVER', 'deleteReportAndMetricsDataJobHandler', '{\"maxdays\":365}', 'DISCARD_LATER', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-07-30 10:34:34', '', '1', '1603036800000', '1603123200000'); +INSERT INTO `xxl_job_info` VALUES ('59', '3', '0 0 0 1 * ?', '删除所有统计数据(Druid)', '2020-07-30 11:32:08', '2020-10-15 17:13:36', 'zhq', '', 'FAILOVER', 'deleteAllReportAndMetricsDataJobHandler', '{\"maxdays\":365}', 'DISCARD_LATER', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:32:08', '', '0', '0', '0'); +INSERT INTO `xxl_job_info` VALUES ('62', '3', '0 0 0 1/1 * ? ', '设置文件存储配额( hos 保留天)', '2020-07-30 11:59:30', '2020-10-15 17:17:44', 'zhq', '', 'FAILOVER', 'deleteFilesJobHandler', '{\"maxdays\":365}', 'DISCARD_LATER', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:59:30', '', '1', '1603036800000', '1603123200000'); +INSERT INTO `xxl_job_info` VALUES ('63', '3', '0 0 0 1 * ?', '删除所有文件(hos)', '2020-07-30 11:59:43', '2020-10-15 17:17:44', 'zhq', '', 'FAILOVER', 'deleteAllFilesJobHandler', '{\"maxdays\":365}', 'DISCARD_LATER', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:59:43', '', '0', '0', '0'); +INSERT INTO `xxl_job_info` VALUES ('64', '3', '0 0 0 1 * ?', '删除所有原始日志(ClickHouse)', '2020-07-30 13:47:25', '2020-10-15 18:26:17', 'zhq', '', 'FAILOVER', 'deleteAllTrafficDataJobHandler', '{\"maxdays\":29}', 'DISCARD_LATER', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-07-30 13:47:25', '', '0', '0', '0'); +INSERT INTO `xxl_job_info` VALUES ('65', '3', '0 0 0 1/1 * ? ', '设置原始日志存储配额（ClickHouse保留天）', '2020-07-30 13:47:38', '2020-10-15 18:27:08', 'zhq', '', 'FAILOVER', 'deleteTrafficDataJobHandler', '{\"maxdays\":30}', 'DISCARD_LATER', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-07-30 13:47:38', '', '1', '1603036800000', '1603123200000'); +INSERT INTO `xxl_job_info` VALUES ('66', '3', '0 0/5 * * * ?', 'Traffic-Top-Client-Ip', '2020-09-23 14:43:56', '2021-01-02 15:30:48', 'galaxy', '', 'FAILOVER', 'httpToKafkaJobHandler', '{\r\n \"url\": \"http://{{ gateway_keepalive_host }}:8089/druid/v2/sql\",\r\n \"method\": \"post\",\r\n \"requestBody\": {\r\n \"query\": \"(SELECT common_client_ip AS source, sum( session_num ) AS session_num,sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num, sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'packets\' as order_by FROM client_internal_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND (common_l4_protocol = \'IPv6_TCP\' OR common_l4_protocol = \'IPv4_TCP\') GROUP BY common_client_ip ORDER BY c2s_pkt_num+s2c_pkt_num DESC LIMIT 1000) UNION ALL (SELECT common_client_ip AS source, sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num, sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num,TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'bytes\' as order_by FROM client_internal_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND (common_l4_protocol = \'IPv6_TCP\' OR common_l4_protocol = \'IPv4_TCP\') GROUP BY common_client_ip ORDER BY c2s_byte_num+s2c_byte_num DESC LIMIT 1000) UNION ALL (SELECT common_client_ip AS source, sum( session_num ) AS session_num,sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num, sum( c2s_pkt_num ) AS c2s_pkt_num, sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time, \'sessions\' as order_by FROM client_internal_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND (common_l4_protocol = \'IPv6_TCP\' OR common_l4_protocol = \'IPv4_TCP\') GROUP BY common_client_ip ORDER BY session_num DESC LIMIT 1000)\",\r\n \"context\": {\r\n \"skipEmptyBuckets\": \"false\"\r\n },\r\n \"resultFormat\": \"object\"\r\n },\r\n \"resultKey\": \"\",\r\n \"topic\": \"TOP-CLIENT-IP-LOG\"\r\n}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-09-23 14:43:56', '', '1', '1609739400000', '1609739700000'); +INSERT INTO `xxl_job_info` VALUES ('67', '3', '0 0/5 * * * ?', 'Traffic-Top-Server-Ip', '2020-09-23 14:46:37', '2021-01-02 15:30:21', 'galaxy', '', 'FAILOVER', 'httpToKafkaJobHandler', '{\r\n \"url\": \"http://{{ gateway_keepalive_host }}:8089/druid/v2/sql\",\r\n \"method\": \"post\",\r\n \"requestBody\": {\r\n \"query\": \"(SELECT common_server_ip AS destination, sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num,TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'packets\' as order_by FROM server_external_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND (common_l4_protocol = \'IPv6_TCP\' OR common_l4_protocol = \'IPv4_TCP\') GROUP BY common_server_ip ORDER BY c2s_pkt_num+s2c_pkt_num DESC LIMIT 1000) UNION ALL (SELECT common_server_ip AS destination, sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'bytes\' as order_by FROM server_external_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND (common_l4_protocol = \'IPv6_TCP\' OR common_l4_protocol = \'IPv4_TCP\') GROUP BY common_server_ip ORDER BY c2s_byte_num+s2c_byte_num DESC LIMIT 1000) UNION ALL (SELECT common_server_ip AS destination, sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num, sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'sessions\' as order_by FROM server_external_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND (common_l4_protocol = \'IPv6_TCP\' OR common_l4_protocol = \'IPv4_TCP\') GROUP BY common_server_ip ORDER BY session_num DESC LIMIT 1000)\",\r\n \"context\": {\r\n \"skipEmptyBuckets\": \"false\"\r\n },\r\n \"resultFormat\": \"object\"\r\n },\r\n \"resultKey\": \"\",\r\n \"topic\": \"TOP-SERVER-IP-LOG\"\r\n}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-09-23 14:46:37', '', '1', '1609739400000', '1609739700000'); +INSERT INTO `xxl_job_info` VALUES ('68', '3', '0 0 0 1/1 * ? ', '设置Druid hot日志保留天', '2020-09-25 19:29:09', '2020-09-28 11:36:17', 'zhq', '', 'FAILOVER', 'deleteDruidHotLogJobHandler', '{\"maxdays\":1}', 'DISCARD_LATER', '0', '2', 'BEAN', '', 'GLUE代码初始化', '2020-09-25 19:29:09', '', '1', '1603036800000', '1603123200000'); + +-- ---------------------------- +-- Table structure for `xxl_job_lock` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_lock`; +CREATE TABLE `xxl_job_lock` ( + `lock_name` varchar(50) NOT NULL COMMENT '锁名称', + PRIMARY KEY (`lock_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- +-- Records of xxl_job_lock +-- ---------------------------- +INSERT INTO `xxl_job_lock` VALUES ('schedule_lock'); + +-- ---------------------------- +-- Table structure for `xxl_job_log` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_log`; +CREATE TABLE `xxl_job_log` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `job_group` int(11) NOT NULL COMMENT '执行器主键ID', + `job_id` int(11) NOT NULL COMMENT '任务，主键ID', + `executor_address` varchar(255) DEFAULT NULL COMMENT '执行器地址，本次执行的地址', + `executor_handler` varchar(255) DEFAULT NULL COMMENT '执行器任务handler', + `executor_param` varchar(4000) DEFAULT '' COMMENT '执行器任务参数', + `executor_sharding_param` varchar(20) DEFAULT NULL COMMENT '执行器任务分片参数，格式如 1/2', + `executor_fail_retry_count` int(11) NOT NULL DEFAULT 0 COMMENT '失败重试次数', + `trigger_time` datetime DEFAULT NULL COMMENT '调度-时间', + `trigger_code` int(11) NOT NULL COMMENT '调度-结果', + `trigger_msg` text DEFAULT NULL COMMENT '调度-日志', + `handle_time` datetime DEFAULT NULL COMMENT '执行-时间', + `handle_code` int(11) NOT NULL COMMENT '执行-状态', + `handle_msg` text DEFAULT NULL COMMENT '执行-日志', + `alarm_status` tinyint(4) NOT NULL DEFAULT 0 COMMENT '告警状态：0-默认、1-无需告警、2-告警成功、3-告警失败', + PRIMARY KEY (`id`), + KEY `I_trigger_time` (`trigger_time`), + KEY `I_handle_code` (`handle_code`) +) ENGINE=InnoDB AUTO_INCREMENT=1715972 DEFAULT CHARSET=utf8mb4; + + +-- ---------------------------- +-- Table structure for `xxl_job_log_report` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_log_report`; +CREATE TABLE `xxl_job_log_report` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `trigger_day` datetime DEFAULT NULL COMMENT '调度-时间', + `running_count` int(11) NOT NULL DEFAULT 0 COMMENT '运行中-日志数量', + `suc_count` int(11) NOT NULL DEFAULT 0 COMMENT '执行成功-日志数量', + `fail_count` int(11) NOT NULL DEFAULT 0 COMMENT '执行失败-日志数量', + PRIMARY KEY (`id`), + UNIQUE KEY `i_trigger_day` (`trigger_day`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=181 DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- + +-- ---------------------------- +-- Table structure for `xxl_job_logglue` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_logglue`; +CREATE TABLE `xxl_job_logglue` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `job_id` int(11) NOT NULL COMMENT '任务，主键ID', + `glue_type` varchar(50) DEFAULT NULL COMMENT 'GLUE类型', + `glue_source` mediumtext DEFAULT NULL COMMENT 'GLUE源代码', + `glue_remark` varchar(128) NOT NULL COMMENT 'GLUE备注', + `add_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- +-- Records of xxl_job_logglue +-- ---------------------------- + +-- ---------------------------- +-- Table structure for `xxl_job_registry` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_registry`; +CREATE TABLE `xxl_job_registry` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `registry_group` varchar(50) NOT NULL, + `registry_key` varchar(255) NOT NULL, + `registry_value` varchar(255) NOT NULL, + `update_time` datetime DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `i_g_k_v` (`registry_group`,`registry_key`,`registry_value`) +) ENGINE=InnoDB AUTO_INCREMENT=235 DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- +-- Table structure for `xxl_job_user` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_user`; +CREATE TABLE `xxl_job_user` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `username` varchar(50) NOT NULL COMMENT '账号', + `password` varchar(50) NOT NULL COMMENT '密码', + `role` tinyint(4) NOT NULL COMMENT '角色：0-普通用户、1-管理员', + `permission` varchar(255) DEFAULT NULL COMMENT '权限：执行器ID列表，多个逗号分割', + PRIMARY KEY (`id`), + UNIQUE KEY `i_username` (`username`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- +-- Records of xxl_job_user +-- ---------------------------- +INSERT INTO `xxl_job_user` VALUES ('1', 'admin', 'fea191a3fdd9f68503f9fa0e8d0293ab', '1', null); +INSERT INTO `xxl_job_user` VALUES ('2', 'query', '44f057733c7a286299a972f6c0c239af', '0', '3'); diff --git a/parcels/roles/init/hbase/tasks/main.yml b/parcels/roles/init/hbase/tasks/main.yml new file mode 100644 index 0000000..fa1022c --- /dev/null +++ b/parcels/roles/init/hbase/tasks/main.yml @@ -0,0 +1,22 @@ +- name: get HBase master_ip + shell: echo "{{ ansible_play_hosts }}" | grep -E -o "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+" | head -n 1 + register: master_out + +- name: master_ip to ansible variable + set_fact: master_ip="{{master_out.stdout}}" + +#复制tar到目标服务器 +- name: copy spark config files + template: + src: 'create-hbase-table.sh.j2' + dest: '{{ install_path }}/{{ hbase_version }}/bin/create-hbase-table.sh' + mode: 0755 + force: true + backup: yes + +- name: create hbase table + shell: cd {{ install_path }}/{{ hbase_version }}/bin/ && ./create-hbase-table.sh + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + diff --git a/parcels/roles/init/hbase/templates/create-hbase-table.sh.j2 b/parcels/roles/init/hbase/templates/create-hbase-table.sh.j2 new file mode 100644 index 0000000..96eca35 --- /dev/null +++ b/parcels/roles/init/hbase/templates/create-hbase-table.sh.j2 @@ -0,0 +1,14 @@ +#!/bin/bash + +source /etc/profile + +exec {{ install_path }}/{{ hbase_version }}/bin/hbase shell <<EOF +create_namespace 'tsg' +create_namespace 'sub' + +create 'tsg:report_result', {NAME => 'response', VERSIONS => 1,COMPRESSION => 'GZ',IS_MOB => true, MOB_THRESHOLD => 0}, {NAME => 'detail',COMPRESSION => 'GZ',VERSIONS => 1} + +create 'sub:subscriber_info', {NAME => 'subscriber_id', VERSIONS => 1} + +list +EOF diff --git a/parcels/roles/init/kafka/tasks/init_kafka.yml b/parcels/roles/init/kafka/tasks/init_kafka.yml new file mode 100644 index 0000000..156aa7a --- /dev/null +++ b/parcels/roles/init/kafka/tasks/init_kafka.yml @@ -0,0 +1,39 @@ +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181" + when: '(groups.zookeeper|length) == 3' + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181,{{groups.zookeeper[3]}}:2181,{{groups.zookeeper[4]}}:2181" + when: '(groups.zookeeper|length) >= 5' + +#- name: get kafka first ip +# shell: echo "{{ ansible_play_hosts }}" | grep -E -o "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+" | head -n 1 +# register: master_out +# run_once: true +# delegate_to: 127.0.0.1 + +- name: master_ip to ansible variable + set_fact: master_ip="{{groups.kafka[0]}}" + +#获取slaves列表 +#- name: get kafka nodes num +# shell: echo "{{ ansible_play_hosts }}" | grep -E -o "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+" | wc -l +# register: nums_out +# run_once: true +# delegate_to: 127.0.0.1 + +- name: partion_nums to ansible variable + set_fact: partion_nums="{{groups.kafka|length}}" + +- name: copy create_topic.sh + template: + src: 'create_topic.sh.j2' + dest: '{{ install_path }}/{{ kafka_version }}/bin/create_topic.sh' + mode: 0755 + backup: yes + +- name: create kafka topic + shell: cd {{ install_path }}/{{ kafka_version }}/bin && sh create_topic.sh + run_once: true + delegate_to: "{{ master_ip }}" + diff --git a/parcels/roles/init/kafka/tasks/main.yml b/parcels/roles/init/kafka/tasks/main.yml new file mode 100644 index 0000000..ac45212 --- /dev/null +++ b/parcels/roles/init/kafka/tasks/main.yml @@ -0,0 +1 @@ +- include: init_kafka.yml diff --git a/parcels/roles/init/kafka/templates/create_topic.sh.j2 b/parcels/roles/init/kafka/templates/create_topic.sh.j2 new file mode 100644 index 0000000..5fe24b2 --- /dev/null +++ b/parcels/roles/init/kafka/templates/create_topic.sh.j2 @@ -0,0 +1,62 @@ +#!/bin/bash + +source /etc/profile + +function createTopic(){ +replication=$1 +partion_nums=$2 +topic_name=$3 + +num=`kafka-topics.sh '--list' '--zookeeper' '{{ zookeeper_servers }}/kafka' | grep $topic_name | wc -l` +if [ $num -eq "0" ];then +kafka-topics.sh '--create' '--zookeeper' '{{ zookeeper_servers }}/kafka' '--replication-factor' $replication '--partitions' $partion_nums '--topic' $topic_name +fi +} + +#原始日志 +createTopic 2 {{ partion_nums }} SECURITY-EVENT-LOG +createTopic 2 {{ partion_nums }} PROXY-EVENT-LOG +createTopic 2 {{ partion_nums }} RADIUS-RECORD-LOG +createTopic 2 {{ partion_nums }} SYS-PACKET-CAPTURE-LOG +createTopic 2 {{ partion_nums }} ACTIVE-DEFENCE-EVENT-LOG +createTopic 2 {{ partion_nums }} VOIP-RECORD-LOG +createTopic 2 {{ partion_nums }} GTPC-RECORD-LOG +createTopic 2 {{ partion_nums }} DOS-SKETCH-LOG + +createTopic 1 {{ partion_nums }} CONNECTION-RECORD-LOG +createTopic 1 {{ partion_nums }} LIVE-SESSION-RECORD-LOG +createTopic 1 {{ partion_nums }} TRANSACTION-RECORD-LOG + +#补全日志 +createTopic 2 {{ partion_nums }} SECURITY-EVENT-COMPLETED-LOG +createTopic 2 {{ partion_nums }} PROXY-EVENT-COMPLETED-LOG +createTopic 2 {{ partion_nums }} RADIUS-RECORD-COMPLETED-LOG +createTopic 2 {{ partion_nums }} SYS-PACKET-CAPTURE-COMPLETED-LOG +createTopic 2 {{ partion_nums }} ACTIVE-DEFENCE-EVENT-COMPLETED-LOG +createTopic 2 {{ partion_nums }} VOIP-RECORD-COMPLETED-LOG +createTopic 2 {{ partion_nums }} VOIP-CONVERSATION-RECORD-LOG +createTopic 2 {{ partion_nums }} GTPC-RECORD-COMPLETED-LOG + +createTopic 1 {{ partion_nums }} CONNECTION-RECORD-COMPLETED-LOG +createTopic 1 {{ partion_nums }} TRANSACTION-RECORD-COMPLETED-LOG +createTopic 1 {{ partion_nums }} LIVE-SESSION-RECORD-COMPLETED-LOG + +#统计日志 +createTopic 2 {{ partion_nums }} TRAFFIC-METRICS-LOG +createTopic 2 {{ partion_nums }} SYS-STORAGE-LOG +createTopic 2 {{ partion_nums }} TOP-EXTERNAL-HOST-LOG +createTopic 2 {{ partion_nums }} TOP-INTERNAL-HOST-LOG +createTopic 2 {{ partion_nums }} TOP-USER-LOG +createTopic 2 {{ partion_nums }} TOP-DOMAIN-RECOMMEND-DAILY-LOG +createTopic 2 {{ partion_nums }} TOP-WEBSITE-URLS-DAILY-LOG +createTopic 2 {{ partion_nums }} TOP-WEBSITE-DOMAIN-LOG +createTopic 2 {{ partion_nums }} TOP-URLS-LOG +createTopic 2 {{ partion_nums }} TOP-CLIENT-IP-LOG +createTopic 2 {{ partion_nums }} TOP-SERVER-IP-LOG +createTopic 2 {{ partion_nums }} RADIUS-ONFF-LOG +createTopic 2 {{ partion_nums }} TRAFFIC-PROTOCOL-STAT-LOG + +#功能端 +createTopic 2 {{ partion_nums }} PXY-EXCH-INTERMEDIA-CERT +createTopic 2 {{ partion_nums }} INTERNAL-PACKET-CAPTURE-LOG +createTopic 2 {{ partion_nums }} INTERNAL-RTP-LOG diff --git a/parcels/roles/init/spark/tasks/init_spark.yml b/parcels/roles/init/spark/tasks/init_spark.yml new file mode 100644 index 0000000..e2361f4 --- /dev/null +++ b/parcels/roles/init/spark/tasks/init_spark.yml @@ -0,0 +1,46 @@ + +#复制tar到目标服务器 +- name: copy iplearning to {{ install_path }}/{{ spark_version }}/ + copy: + src: '{{ package_path }}/iplearning' + dest: '{{ install_path }}/{{ spark_version }}/' + force: true + backup: yes + +- name: copy spark iplearning shell to {{ install_path }}/{{ spark_version }}/iplearning/ + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + backup: yes + force: true + with_items: + - { src: 'iplearning_monitor.sh.j2', dest: '{{ install_path }}/{{ spark_version }}/iplearning/iplearning_monitor.sh' } + - { src: 'iplearning.sh.j2', dest: '{{ install_path }}/{{ spark_version }}/iplearning/iplearning.sh' } + +- name: get spark master IP + shell: echo "{{ ansible_play_hosts }}" | grep -E -o "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+" | head -n 1 + register: master_out + run_once: true + delegate_to: 127.0.0.1 + +- name: master_ip to ansible variable + set_fact: master_ip="{{master_out.stdout}}" + +- name: copy application.properties to {{ install_path }}/{{ spark_version }}/iplearning/ + template: + src: 'application.properties.j2' + dest: '{{ install_path }}/{{ spark_version }}/iplearning/application.properties' + force: true + +- name: change iplearning.jar config + shell: cd {{ install_path }}/{{ spark_version }}/iplearning/ && chmod +x upconfig.sh && sh upconfig.sh + +- name: + cron: + minute: "0" + hour: "*/1" + job: "sh {{ install_path }}/{{ spark_version }}/iplearning/iplearning.sh" + state: present + run_once: true + delegate_to: "{{ master_ip }}" + diff --git a/parcels/roles/init/spark/tasks/main.yml b/parcels/roles/init/spark/tasks/main.yml new file mode 100644 index 0000000..1aa6810 --- /dev/null +++ b/parcels/roles/init/spark/tasks/main.yml @@ -0,0 +1 @@ +- include: init_spark.yml diff --git a/parcels/roles/init/spark/templates/application.properties.j2 b/parcels/roles/init/spark/templates/application.properties.j2 new file mode 100644 index 0000000..fd9493f --- /dev/null +++ b/parcels/roles/init/spark/templates/application.properties.j2 @@ -0,0 +1,39 @@ +#spark任务配置 +spark.sql.shuffle.partitions=50 +spark.executor.memory=50g +spark.app.name=spark-iplearning +spark.network.timeout=300s +spark.serializer=org.apache.spark.serializer.KryoSerializer +#spark.serializer=org.apache.spark.serializer.JavaSerializer +master=spark://{{ groups.spark[0] }}:7077 +#spark读取clickhouse配置 +spark.read.clickhouse.url=jdbc:clickhouse://{{ gateway_keepalive_host }}:8123/tsg_galaxy_v3 +spark.read.clickhouse.driver=ru.yandex.clickhouse.ClickHouseDriver +spark.read.clickhouse.user=default +spark.read.clickhouse.password=ceiec2019 +spark.read.clickhouse.numPartitions=5 +spark.read.clickhouse.fetchsize=50000 +spark.read.clickhouse.partitionColumn=LAST_FOUND_TIME +clickhouse.socket.timeout=3600000 +#arangoDB配置 +arangoDB.host={{ groups.arangodb[0] }} +arangoDB.port=8529 +arangoDB.user=upsert +arangoDB.password=ceiec2018 +arangoDB.DB.name=tsg_galaxy_v3 +arangoDB.ttl=3600 + +thread.pool.number=50 + +#读取clickhouse时间范围方式，0：读取过去一小时；1：指定时间范围 +clickhouse.time.limit.type=0 +read.clickhouse.max.time=1600246160 +read.clickhouse.min.time=1597197469 + +arangoDB.read.sepNum=1 +update.arango.batch=10000 + +distinct.client.ip.num=1000 +recent.count.hour=24 + +update.interval=3600 diff --git a/parcels/roles/init/spark/templates/iplearning.sh.j2 b/parcels/roles/init/spark/templates/iplearning.sh.j2 new file mode 100644 index 0000000..954dc8c --- /dev/null +++ b/parcels/roles/init/spark/templates/iplearning.sh.j2 @@ -0,0 +1,32 @@ +#!/bin/bash +source /etc/profile + +jar_name="ip-learning-spark.jar" + +spark_home={{ install_path }}/{{ spark_version }} + +isrun=`ps -ef | grep ip-learning-spark.jar | wc -l` + + +if [[ $isrun >= "1" ]];then +OLD_NUM=`cat $spark_home/iplearning/runnum` +RESTART_NUM=`expr $OLD_NUM + 1` +echo $RESTART_NUM > $spark_home/iplearning/runnum + +if [ $RESTART_NUM -eq "3" ];then +ps aux|grep "ip-learning-spark.jar"|grep -v grep|awk '{print $2}'|xargs kill -9 + +nohup ${spark_home}/bin/spark-submit --class cn.ac.iie.main.IpLearningApplication --driver-java-options "-Dlog4j.configuration=file:${spark_home}/conf/log4j-task.properties" --conf spark.executor.extraJavaOptions="-Dlog4j.configuration=file:${spark_home}/conf/log4j-task.properties" ${spark_home}/iplearning/$jar_name >/dev/null 2>&1 & + +echo 1 > $spark_home/iplearning/runnum +fi + +else + +nohup ${spark_home}/bin/spark-submit --class cn.ac.iie.main.IpLearningApplication --driver-java-options "-Dlog4j.configuration=file:${spark_home}/conf/log4j-task.properties" --conf spark.executor.extraJavaOptions="-Dlog4j.configuration=file:${spark_home}/conf/log4j-task.properties" ${spark_home}/iplearning/$jar_name >/dev/null 2>&1 & + +echo 1 > $spark_home/iplearning/runnum + +nohup ${spark_home}/iplearning/iplearning_monitor.sh > /dev/null 2>&1 & + +fi diff --git a/parcels/roles/init/spark/templates/iplearning_monitor.sh.j2 b/parcels/roles/init/spark/templates/iplearning_monitor.sh.j2 new file mode 100644 index 0000000..06df1e6 --- /dev/null +++ b/parcels/roles/init/spark/templates/iplearning_monitor.sh.j2 @@ -0,0 +1,55 @@ +#! /bin/bash + +source /etc/profile + +#iplearning 日志文件目录 +iplearning_log_file='{{ install_path }}/{{ spark_version }}/logs/ip-learning-application.log' +iplearning_monitor_prom_file="/opt/nezha/node_exporter/prom/iplearning_monitor.prom" + +#arangoDB R_LOCATE_FQDN2IP数量 +fqdn_locate_ip_arango_count=`sed -n '/RETURN LENGTH(R_LOCATE_FQDN2IP)/p' $iplearning_log_file | awk -v FS='：' '{print $2}' | awk '{print $1}' | awk 'END {print}'` +echo "fqdn_locate_ip_arango_count $fqdn_locate_ip_arango_count" > $iplearning_monitor_prom_file + +#读取arangoDB R_LOCATE_FQDN2IP耗时 +read_fqdn_locate_ip_arango_time=`sed -n '/读取R_LOCATE_FQDN2IP arangoDB 共耗时/p' $iplearning_log_file | awk -v FS='：' '{print $2}' | awk '{print $1}' | awk 'END {print}'` +echo "read_fqdn_locate_ip_arango_time $read_fqdn_locate_ip_arango_time" >> $iplearning_monitor_prom_file + +#clickhouse fqdn-locate-ip边关系数量 +fqdn_locate_ip_ck_count=`sed -n '/读取R_LOCATE_FQDN2IP clickhouse成功/p' $iplearning_log_file | awk -v FS='：' '{print $2}' | awk '{print $1}' | awk 'END {print}'` +echo fqdn_locate_ip_ck_count $fqdn_locate_ip_ck_count >> $iplearning_monitor_prom_file + +#更新R_LOCATE_FQDN2IP耗时 +update_fqdn_locate_ip_time=`sed -n '/更新R_LOCATE_FQDN2IP 时间/p' $iplearning_log_file | awk -v FS='：' '{print $2}' | awk '{print $1}' | awk 'END {print}'` +echo update_fqdn_locate_ip_time $update_fqdn_locate_ip_time >> $iplearning_monitor_prom_file + +#arangoDB SUBSCRIBER数量 +subid_arango_count=`sed -n '/RETURN LENGTH(SUBSCRIBER)/p' $iplearning_log_file | awk -v FS='：' '{print $2}' | awk '{print $1}' | awk 'END {print}'` +echo subid_arango_count $subid_arango_count >> $iplearning_monitor_prom_file + +#读取arangoDB SUBSCRIBER耗时 +read_subid_arango_time=`sed -n '/读取SUBSCRIBER arangoDB 共耗时/p' $iplearning_log_file | awk -v FS='：' '{print $2}' | awk '{print $1}' | awk 'END {print}'` +echo read_subid_arango_time $read_subid_arango_time >> $iplearning_monitor_prom_file + +#clickhouse subscriberid点数量 +subid_ck_count=`sed -n '/读取SUBSCRIBER clickhouse成功/p' $iplearning_log_file | awk -v FS='：' '{print $2}' | awk '{print $1}' | awk 'END {print}'` +echo subid_ck_count $subid_ck_count >> $iplearning_monitor_prom_file + +#更新SUBSCRIBER耗时 +update_subid_time=`sed -n '/更新SUBSCRIBER 时间/p' $iplearning_log_file | awk -v FS='：' '{print $2}' | awk '{print $1}' | awk 'END {print}'` +echo update_subid_time $update_subid_time >> $iplearning_monitor_prom_file + +#arangoDB R_LOCATE_SUBSCRIBER2IP数量 +subid_locate_ip_arango_count=`sed -n '/RETURN LENGTH(R_LOCATE_SUBSCRIBER2IP)/p' $iplearning_log_file | awk -v FS='：' '{print $2}' | awk '{print $1}' | awk 'END {print}'` +echo subid_locate_ip_arango_count $subid_locate_ip_arango_count >> $iplearning_monitor_prom_file + +#读取arangoDB R_LOCATE_SUBSCRIBER2IP耗时 +read_subid_locate_ip_arango_time=`sed -n '/读取R_LOCATE_SUBSCRIBER2IP arangoDB 共耗时/p' $iplearning_log_file | awk -v FS='：' '{print $2}' | awk '{print $1}' | awk 'END {print}'` +echo read_subid_locate_ip_arango_time $read_subid_locate_ip_arango_time >> $iplearning_monitor_prom_file + +#clickhouse subscriberid-locate-ip边关系数量 +subid_locate_ip_ck_count=`sed -n '/读取R_LOCATE_SUBSCRIBER2IP clickhouse成功/p' $iplearning_log_file | awk -v FS='：' '{print $2}' | awk '{print $1}' | awk 'END {print}'` +echo subid_locate_ip_ck_count $subid_locate_ip_ck_count >> $iplearning_monitor_prom_file + +#更新R_LOCATE_SUBSCRIBER2IP耗时 +update_subid_locate_ip_time=`sed -n '/更新R_LOCATE_SUBSCRIBER2IP 时间/p' $iplearning_log_file | awk -v FS='：' '{print $2}' | awk '{print $1}' | awk 'END {print}'` +echo update_subid_locate_ip_time $update_subid_locate_ip_time >> $iplearning_monitor_prom_file diff --git a/parcels/roles/init/storm/tasks/main.yml b/parcels/roles/init/storm/tasks/main.yml new file mode 100644 index 0000000..2305b11 --- /dev/null +++ b/parcels/roles/init/storm/tasks/main.yml @@ -0,0 +1,117 @@ +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181" + when: '(groups.zookeeper|length) == 3' + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181,{{groups.zookeeper[3]}}:2181,{{groups.zookeeper[4]}}:2181" + when: '(groups.zookeeper|length) >= 5' + +- name: master_ip to ansible variable + set_fact: master_ip="{{groups.storm[0]}}" + +#复制任务到目标服务器 +- name: copy topology path:{{ install_path }} + copy: + src: '{{ package_path }}/topology' + dest: '{{ install_path }}/' + force: true + backup: yes + +#解压zip +#- name: unpack topology.zip +# unarchive: +# src: 'topology.zip' +# dest: '{{ install_path }}/' + +- name: copy topology config to files + template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + force: true + with_items: + - { src: 'ACTIVE-DEFENCE-EVENT-COMPLETED-LOG.j2', dest: '{{ install_path }}/topology/completion/config/ACTIVE-DEFENCE-EVENT-COMPLETED-LOG' } + - { src: 'CONNECTION-RECORD-COMPLETED-LOG.j2', dest: '{{ install_path }}/topology/completion/config/CONNECTION-RECORD-COMPLETED-LOG' } + - { src: 'PROXY-EVENT-COMPLETED-LOG.j2', dest: '{{ install_path }}/topology/completion/config/PROXY-EVENT-COMPLETED-LOG' } + - { src: 'RADIUS-RECORD-COMPLETED-LOG.j2', dest: '{{ install_path }}/topology/completion/config/RADIUS-RECORD-COMPLETED-LOG' } + - { src: 'SECURITY-EVENT-COMPLETED-LOG.j2', dest: '{{ install_path }}/topology/completion/config/SECURITY-EVENT-COMPLETED-LOG' } + - { src: 'SYS-PACKET-CAPTURE-COMPLETED-LOG.j2', dest: '{{ install_path }}/topology/completion/config/SYS-PACKET-CAPTURE-COMPLETED-LOG' } + - { src: 'GTPC-RECORD-COMPLETED-LOG.j2', dest: '{{ install_path }}/topology/completion/config/GTPC-RECORD-COMPLETED-LOG' } + - { src: 'VOIP-RECORD-COMPLETED-LOG.j2', dest: '{{ install_path }}/topology/completion/config/VOIP-RECORD-COMPLETED-LOG' } + - { src: 'LIVE-SESSION-RECORD-COMPLETED-LOG.j2', dest: '{{ install_path }}/topology/completion/config/LIVE-SESSION-RECORD-COMPLETED-LOG' } + - { src: 'TRANSACTION-RECORD-COMPLETED-LOG.j2', dest: '{{ install_path }}/topology/completion/config/TRANSACTION-RECORD-COMPLETED-LOG' } + - { src: 'LIVE-CHARTS_CONNECTION-RECORD-LOG.j2', dest: '{{ install_path }}/topology/livecharts/config/LIVE-CHARTS_CONNECTION-RECORD-LOG' } + - { src: 'LIVE-CHARTS_LIVE-SESSION-RECORD-LOG.j2', dest: '{{ install_path }}/topology/livecharts/config/LIVE-CHARTS_LIVE-SESSION-RECORD-LOG' } + - { src: 'RADIUS-ON-OFF.j2', dest: '{{ install_path }}/topology/radiusOnOff/config/RADIUS-ON-OFF' } + - { src: 'SUBSCRUBER-HBASE.j2', dest: '{{ install_path }}/topology/subscriberHbase/config/SUBSCRUBER-HBASE' } + - { src: 'VOIP-RELATION.j2', dest: '{{ install_path }}/topology/voipRelation/config/VOIP-RELATION' } +# - { src: 'SUBSCRUBER-HBASE-SYNC.j2', dest: '{{ install_path }}/topology/subscriberHbaseSync/config/SUBSCRUBER-HBASE-SYNC' } + +- name: stop storm completion topology + shell: cd {{ install_path }}/topology/completion && chmod +x *.sh && ./stop.sh config/ + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + +- name: stop storm livecharts topology + shell: cd {{ install_path }}/topology/livecharts && chmod +x *.sh && ./stop.sh config/ + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + +- name: stop storm radiusOnOff topology + shell: cd {{ install_path }}/topology/radiusOnOff && chmod +x *.sh && ./stop.sh config/ + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + +- name: stop storm subscriberHbase topology + shell: cd {{ install_path }}/topology/subscriberHbase && chmod +x *.sh && ./stop.sh config/ + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + +- name: stop storm voiprelation topology + shell: cd {{ install_path }}/topology/voipRelation && chmod +x *.sh && ./stop.sh config/ + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + +- name: start storm completion topology + shell: cd {{ install_path }}/topology/completion && chmod +x *.sh && ./start.sh config/ + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + +- name: start storm livecharts topology + shell: cd {{ install_path }}/topology/livecharts && chmod +x *.sh && ./start.sh config/ + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + +- name: start storm radiusOnOff topology + shell: cd {{ install_path }}/topology/radiusOnOff && chmod +x *.sh && ./start.sh config/ + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + +- name: start storm subscriberHbase topology + shell: cd {{ install_path }}/topology/subscriberHbase && chmod +x *.sh && ./start.sh config/ + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + +- name: start storm voiprelation topology + shell: cd {{ install_path }}/topology/voipRelation && chmod +x *.sh && ./start.sh config/ + run_once: true + delegate_facts: true + delegate_to: "{{ master_ip }}" + + + +#- name: Ansible delete topology.zip +# file: +# path: "{{ install_path }}/topology.zip" +# state: absent + + + diff --git a/parcels/roles/init/storm/templates/ACTIVE-DEFENCE-EVENT-COMPLETED-LOG.j2 b/parcels/roles/init/storm/templates/ACTIVE-DEFENCE-EVENT-COMPLETED-LOG.j2 new file mode 100644 index 0000000..a66e3d8 --- /dev/null +++ b/parcels/roles/init/storm/templates/ACTIVE-DEFENCE-EVENT-COMPLETED-LOG.j2 @@ -0,0 +1,101 @@ +#--------------------------------地址配置------------------------------# + +#管理kafka地址 +input.kafka.servers={{ consumer_kafka_servers }} + +#管理输出kafka地址 +output.kafka.servers={{ producer_kafka_servers }} + +#zookeeper 地址 用于配置log_id +zookeeper.servers={{ zookeeper_servers }} + +#hbase zookeeper地址 用于连接HBase +hbase.zookeeper.servers={{ zookeeper_servers }} + +#--------------------------------HTTP/定位库------------------------------# +#定位库地址 +ip.library={{ install_path }}/topology/dat/ + +#网关的schema位置 +schema.http=http://{{ gateway_keepalive_host }}:9999/metadata/schema/v1/fields/active_defence_event_log + +#网关APP_ID 获取接口 +app.id.http=http://{{ gateway_keepalive_host }}:9999/open-api/appDicList + +#--------------------------------Kafka消费组信息------------------------------# + +#kafka 接收数据topic +kafka.topic=ACTIVE-DEFENCE-EVENT-LOG + +#补全数据 输出 topic +results.output.topic=ACTIVE-DEFENCE-EVENT-COMPLETED-LOG + +#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； +group.id=active-defence-log-20210124-1 + +#生产者压缩模式 none or snappy +producer.kafka.compression.type=none + +#生产者ack +producer.ack=1 + +#latest/earliest 从当前消 or 从头消费 +auto.offset.reset=latest + +#接收自kafka的消费者 client-id +consumer.client.id=consumer-active-defence + +#回写给kafka的生产者 client-id +producer.client.id=producer-active-defence + +#--------------------------------topology配置------------------------------# + +#storm topology workers +topology.workers=1 + +#spout并行度 建议与kafka分区数相同 +spout.parallelism=3 + +#处理补全操作的bolt并行度-worker的倍数 +completion.bolt.parallelism=3 + +#写入kafka的并行度10 +kafka.bolt.parallelism=3 + +#数据中心（UID） +data.center.id.num=14 + +#hbase 更新时间，如填写0则不更新缓存 +hbase.tick.tuple.freq.secs=180 + +#app_id 更新时间，如填写0则不更新缓存 +app.tick.tuple.freq.secs=0 + +#--------------------------------默认值配置------------------------------# + +#当bolt性能受限时，限制spout接收速度，理论看ack开启才有效 +topology.config.max.spout.pending=70000 + +#hbase table name +hbase.table.name=subscriber_info + +#ack设置 1启动ack 0不启动ack +topology.num.acks=1 + +#kafka批量条数 +batch.insert.num=500 + +#tick时钟频率 +topology.tick.tuple.freq.secs=5 + +#spout接收睡眠时间 +topology.spout.sleep.time=1 + +#邮件默认编码 +mail.default.charset=UTF-8 + +#是否需要补全，yes补全/no日志原样输出 +log.need.complete=yes + +#补全校验类型 0 强制类型校验；1 弱类型校验；2 不校验 +complete.check.type=1 diff --git a/parcels/roles/init/storm/templates/CONNECTION-RECORD-COMPLETED-LOG.j2 b/parcels/roles/init/storm/templates/CONNECTION-RECORD-COMPLETED-LOG.j2 new file mode 100644 index 0000000..aab6e56 --- /dev/null +++ b/parcels/roles/init/storm/templates/CONNECTION-RECORD-COMPLETED-LOG.j2 @@ -0,0 +1,101 @@ +#--------------------------------地址配置------------------------------# + +#管理kafka地址 +input.kafka.servers={{ consumer_kafka_servers }} + +#管理输出kafka地址 +output.kafka.servers={{ producer_kafka_servers }} + +#zookeeper 地址 用于配置log_id +zookeeper.servers={{ zookeeper_servers }} + +#hbase zookeeper地址 用于连接HBase +hbase.zookeeper.servers={{ zookeeper_servers }} + +#--------------------------------HTTP/定位库------------------------------# +#定位库地址 +ip.library={{ install_path }}/topology/dat/ + +#网关的schema位置 +schema.http=http://{{ gateway_keepalive_host }}:9999/metadata/schema/v1/fields/connection_record_log + +#网关APP_ID 获取接口 +app.id.http=http://{{ gateway_keepalive_host }}:9999/open-api/appDicList + +#--------------------------------Kafka消费组信息------------------------------# + +#kafka 接收数据topic +kafka.topic=CONNECTION-RECORD-LOG + +#补全数据 输出 topic +results.output.topic=CONNECTION-RECORD-COMPLETED-LOG + +#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； +group.id=connection-record-log-20210510-1 + +#生产者压缩模式 none or snappy +producer.kafka.compression.type=none + +#生产者ack +producer.ack=1 + +#latest/earliest 从当前消 or 从头消费 +auto.offset.reset=latest + +#接收自kafka的消费者 client-id +consumer.client.id=consumer-connection-record + +#回写给kafka的生产者 client-id +producer.client.id=producer-connection-record + +#--------------------------------topology配置------------------------------# + +#storm topology workers +topology.workers=3 + +#spout并行度 建议与kafka分区数相同 +spout.parallelism=3 + +#处理补全操作的bolt并行度-worker的倍数 +completion.bolt.parallelism=12 + +#写入kafka的并行度10 +kafka.bolt.parallelism=6 + +#数据中心（UID） +data.center.id.num=15 + +#hbase 更新时间，如填写0则不更新缓存 +hbase.tick.tuple.freq.secs=180 + +#app_id 更新时间，如填写0则不更新缓存 +app.tick.tuple.freq.secs=0 + +#--------------------------------默认值配置------------------------------# + +#当bolt性能受限时，限制spout接收速度，理论看ack开启才有效 +topology.config.max.spout.pending=70000 + +#hbase table name +hbase.table.name=subscriber_info + +#ack设置 1启动ack 0不启动ack +topology.num.acks=3 + +#kafka批量条数 +batch.insert.num=500 + +#tick时钟频率 +topology.tick.tuple.freq.secs=5 + +#spout接收睡眠时间 +topology.spout.sleep.time=1 + +#邮件默认编码 +mail.default.charset=UTF-8 + +#是否需要补全，yes补全/no日志原样输出 +log.need.complete=yes + +#补全校验类型 0 强制类型校验；1 弱类型校验；2 不校验 +complete.check.type=1 diff --git a/parcels/roles/init/storm/templates/GTPC-RECORD-COMPLETED-LOG.j2 b/parcels/roles/init/storm/templates/GTPC-RECORD-COMPLETED-LOG.j2 new file mode 100644 index 0000000..b18f3dc --- /dev/null +++ b/parcels/roles/init/storm/templates/GTPC-RECORD-COMPLETED-LOG.j2 @@ -0,0 +1,101 @@ +#--------------------------------地址配置------------------------------# + +#管理kafka地址 +input.kafka.servers={{ consumer_kafka_servers }} + +#管理输出kafka地址 +output.kafka.servers={{ producer_kafka_servers }} + +#zookeeper 地址 用于配置log_id +zookeeper.servers={{ zookeeper_servers }} + +#hbase zookeeper地址 用于连接HBase +hbase.zookeeper.servers={{ zookeeper_servers }} + +#--------------------------------HTTP/定位库------------------------------# +#定位库地址 +ip.library={{ install_path }}/topology/dat/ + +#网关的schema位置 +schema.http=http://{{ gateway_keepalive_host }}:9999/metadata/schema/v1/fields/gtpc_record_log + +#网关APP_ID 获取接口 +app.id.http=http://{{ gateway_keepalive_host }}:9999/open-api/appDicList + +#--------------------------------Kafka消费组信息------------------------------# + +#kafka 接收数据topic +kafka.topic=GTPC-RECORD-LOG + +#补全数据 输出 topic +results.output.topic=GTPC-RECORD-COMPLETED-LOG + +#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； +group.id=gtpc-record-log-20210510-A + +#生产者压缩模式 none or snappy +producer.kafka.compression.type=none + +#生产者ack +producer.ack=1 + +#latest/earliest 从当前消 or 从头消费 +auto.offset.reset=latest + +#接收自kafka的消费者 client-id +consumer.client.id=consumer-gtpc-record + +#回写给kafka的生产者 client-id +producer.client.id=producer-gtpc-record + +#--------------------------------topology配置------------------------------# + +#storm topology workers +topology.workers=1 + +#spout并行度 建议与kafka分区数相同 +spout.parallelism=3 + +#处理补全操作的bolt并行度-worker的倍数 +completion.bolt.parallelism=6 + +#写入kafka的并行度10 +kafka.bolt.parallelism=6 + +#数据中心（UID） +data.center.id.num=9 + +#hbase 更新时间，如填写0则不更新缓存 +hbase.tick.tuple.freq.secs=180 + +#app_id 更新时间，如填写0则不更新缓存 +app.tick.tuple.freq.secs=0 + +#--------------------------------默认值配置------------------------------# + +#当bolt性能受限时，限制spout接收速度，理论看ack开启才有效 +topology.config.max.spout.pending=70000 + +#hbase table name +hbase.table.name=subscriber_info + +#ack设置 1启动ack 0不启动ack +topology.num.acks=1 + +#kafka批量条数 +batch.insert.num=500 + +#tick时钟频率 +topology.tick.tuple.freq.secs=5 + +#spout接收睡眠时间 +topology.spout.sleep.time=1 + +#邮件默认编码 +mail.default.charset=UTF-8 + +#是否需要补全，yes补全/no日志原样输出 +log.need.complete=yes + +#补全校验类型 0 强制类型校验；1 弱类型校验；2 不校验 +complete.check.type=1 diff --git a/parcels/roles/init/storm/templates/LIVE-CHARTS_CONNECTION-RECORD-LOG.j2 b/parcels/roles/init/storm/templates/LIVE-CHARTS_CONNECTION-RECORD-LOG.j2 new file mode 100644 index 0000000..1ddfa5c --- /dev/null +++ b/parcels/roles/init/storm/templates/LIVE-CHARTS_CONNECTION-RECORD-LOG.j2 @@ -0,0 +1,66 @@ +#--------------------------------地址配置------------------------------# +#管理kafka地址 +bootstrap.servers={{ consumer_kafka_servers }} + +#输出topic +results.bootstrap.servers={{ producer_kafka_servers }} + +#--------------------------------Kafka消费组信息------------------------------# +#kafka broker下的topic名称 +kafka.topic=CONNECTION-RECORD-LOG + +#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； +group.id=history-storm-liveCharts-20210802-1 + +#输出topic +results.output.topic=TRAFFIC-PROTOCOL-STAT-LOG + +#latest/earliest +auto.offset.reset=latest + +#接收自kafka的消费者 client-id +consumer.client.id=history-chart-consumer-connection-record + +#回写给kafka的生产者 client-id +producer.client.id=history-chart-producer-connection-record + +#生产者压缩模式 none or snappy +producer.kafka.compression.type=none + +#--------------------------------HTTP/定位库------------------------------# + +#网关的schema位置 +schema.http=http://{{ gateway_keepalive_host }}:9999/metadata/schema/v1/fields/liveChart_history + +#网关的schema位置 +app.id.http=http://{{ gateway_keepalive_host }}:9999/open-api/appDicList + + +#--------------------------------topology配置------------------------------# +#storm topology workers +topology.workers=1 + +#spout并行度 建议与kafka分区数相同 +spout.parallelism=1 + +#拆分bolt并行度 +parse.bolt.parallelism=1 + +#第一次聚合bolt并行度 +calculate.bolt.parallelism=1 + +#写入kafka的并行度10 +kafka.bolt.parallelism=1 + +#聚合时间,单位秒 +first.agg.time=15 + +#更新APP-ID时间 +update.app.id.time=60 + +#--------------------------------默认值配置------------------------------# +#ack设置 1启动ack 0不启动ack +topology.num.acks=1 + +#spout接收睡眠时间 +topology.spout.sleep.time=1 diff --git a/parcels/roles/init/storm/templates/LIVE-CHARTS_LIVE-SESSION-RECORD-LOG.j2 b/parcels/roles/init/storm/templates/LIVE-CHARTS_LIVE-SESSION-RECORD-LOG.j2 new file mode 100644 index 0000000..7a9d454 --- /dev/null +++ b/parcels/roles/init/storm/templates/LIVE-CHARTS_LIVE-SESSION-RECORD-LOG.j2 @@ -0,0 +1,66 @@ +#--------------------------------地址配置------------------------------# +#管理kafka地址 +bootstrap.servers={{ consumer_kafka_servers }} + +#输出topic +results.bootstrap.servers={{ producer_kafka_servers }} + +#--------------------------------Kafka消费组信息------------------------------# +#kafka broker下的topic名称 +kafka.topic=LIVE-SESSION-RECORD-LOG + +#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； +group.id=live-storm-liveCharts-20210802-1 + +#输出topic +results.output.topic=TRAFFIC-PROTOCOL-STAT-LOG + +#latest/earliest +auto.offset.reset=latest + +#接收自kafka的消费者 client-id +consumer.client.id=live-chart-consumer-connection-record + +#回写给kafka的生产者 client-id +producer.client.id=live-chart-producer-connection-record + +#生产者压缩模式 none or snappy +producer.kafka.compression.type=none + +#--------------------------------HTTP/定位库------------------------------# + +#网关的schema位置 +schema.http=http://{{ gateway_keepalive_host }}:9999/metadata/schema/v1/fields/liveChart_live + +#网关的schema位置 +app.id.http=http://{{ gateway_keepalive_host }}:9999/open-api/appDicList + + +#--------------------------------topology配置------------------------------# +#storm topology workers +topology.workers=1 + +#spout并行度 建议与kafka分区数相同 +spout.parallelism=1 + +#拆分bolt并行度 +parse.bolt.parallelism=1 + +#第一次聚合bolt并行度 +calculate.bolt.parallelism=1 + +#写入kafka的并行度10 +kafka.bolt.parallelism=1 + +#聚合时间,单位秒 +first.agg.time=15 + +#更新APP-ID时间 +update.app.id.time=60 + +#--------------------------------默认值配置------------------------------# +#ack设置 1启动ack 0不启动ack +topology.num.acks=1 + +#spout接收睡眠时间 +topology.spout.sleep.time=1 diff --git a/parcels/roles/init/storm/templates/LIVE-SESSION-RECORD-COMPLETED-LOG.j2 b/parcels/roles/init/storm/templates/LIVE-SESSION-RECORD-COMPLETED-LOG.j2 new file mode 100644 index 0000000..de3335c --- /dev/null +++ b/parcels/roles/init/storm/templates/LIVE-SESSION-RECORD-COMPLETED-LOG.j2 @@ -0,0 +1,101 @@ +#--------------------------------地址配置------------------------------# + +#管理kafka地址 +input.kafka.servers={{ consumer_kafka_servers }} + +#管理输出kafka地址 +output.kafka.servers={{ producer_kafka_servers }} + +#zookeeper 地址 用于配置log_id +zookeeper.servers={{ zookeeper_servers }} + +#hbase zookeeper地址 用于连接HBase +hbase.zookeeper.servers={{ zookeeper_servers }} + +#--------------------------------HTTP/定位库------------------------------# +#定位库地址 +ip.library={{ install_path }}/topology/dat/ + +#网关的schema位置 +schema.http=http://{{ gateway_keepalive_host }}:9999/metadata/schema/v1/fields/live_session_record_log + +#网关APP_ID 获取接口 +app.id.http=http://{{ gateway_keepalive_host }}:9999/open-api/appDicList + +#--------------------------------Kafka消费组信息------------------------------# + +#kafka 接收数据topic +kafka.topic=LIVE-SESSION-RECORD-LOG + +#补全数据 输出 topic +results.output.topic=LIVE-SESSION-RECORD-COMPLETED-LOG + +#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； +group.id=live-session-record-log-20210510-A + +#生产者压缩模式 none or snappy +producer.kafka.compression.type=none + +#生产者ack +producer.ack=1 + +#latest/earliest 从当前消 or 从头消费 +auto.offset.reset=latest + +#接收自kafka的消费者 client-id +consumer.client.id=consumer-live-session-record + +#回写给kafka的生产者 client-id +producer.client.id=producer-live-session-record + +#--------------------------------topology配置------------------------------# + +#storm topology workers +topology.workers=1 + +#spout并行度 建议与kafka分区数相同 +spout.parallelism=3 + +#处理补全操作的bolt并行度-worker的倍数 +completion.bolt.parallelism=12 + +#写入kafka的并行度10 +kafka.bolt.parallelism=6 + +#数据中心（UID） +data.center.id.num=7 + +#hbase 更新时间，如填写0则不更新缓存 +hbase.tick.tuple.freq.secs=180 + +#app_id 更新时间，如填写0则不更新缓存 +app.tick.tuple.freq.secs=0 + +#--------------------------------默认值配置------------------------------# + +#当bolt性能受限时，限制spout接收速度，理论看ack开启才有效 +topology.config.max.spout.pending=70000 + +#hbase table name +hbase.table.name=subscriber_info + +#ack设置 1启动ack 0不启动ack +topology.num.acks=1 + +#kafka批量条数 +batch.insert.num=500 + +#tick时钟频率 +topology.tick.tuple.freq.secs=5 + +#spout接收睡眠时间 +topology.spout.sleep.time=1 + +#邮件默认编码 +mail.default.charset=UTF-8 + +#是否需要补全，yes补全/no日志原样输出 +log.need.complete=yes + +#补全校验类型 0 强制类型校验；1 弱类型校验；2 不校验 +complete.check.type=1 diff --git a/parcels/roles/init/storm/templates/PROXY-EVENT-COMPLETED-LOG.j2 b/parcels/roles/init/storm/templates/PROXY-EVENT-COMPLETED-LOG.j2 new file mode 100644 index 0000000..9c14cfe --- /dev/null +++ b/parcels/roles/init/storm/templates/PROXY-EVENT-COMPLETED-LOG.j2 @@ -0,0 +1,101 @@ +#--------------------------------地址配置------------------------------# + +#管理kafka地址 +input.kafka.servers={{ consumer_kafka_servers }} + +#管理输出kafka地址 +output.kafka.servers={{ producer_kafka_servers }} + +#zookeeper 地址 用于配置log_id +zookeeper.servers={{ zookeeper_servers }} + +#hbase zookeeper地址 用于连接HBase +hbase.zookeeper.servers={{ zookeeper_servers }} + +#--------------------------------HTTP/定位库------------------------------# +#定位库地址 +ip.library={{ install_path }}/topology/dat/ + +#网关的schema位置 +schema.http=http://{{ gateway_keepalive_host }}:9999/metadata/schema/v1/fields/proxy_event_log + +#网关APP_ID 获取接口 +app.id.http=http://{{ gateway_keepalive_host }}:9999/open-api/appDicList + +#--------------------------------Kafka消费组信息------------------------------# + +#kafka 接收数据topic +kafka.topic=PROXY-EVENT-LOG + +#补全数据 输出 topic +results.output.topic=PROXY-EVENT-COMPLETED-LOG + +#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； +group.id=proxy-event-log-20210124-1 + +#生产者压缩模式 none or snappy +producer.kafka.compression.type=none + +#生产者ack +producer.ack=1 + +#latest/earliest 从当前消 or 从头消费 +auto.offset.reset=latest + +#接收自kafka的消费者 client-id +consumer.client.id=consumer-proxy-event + +#回写给kafka的生产者 client-id +producer.client.id=producer-proxy-event + +#--------------------------------topology配置------------------------------# + +#storm topology workers +topology.workers=1 + +#spout并行度 建议与kafka分区数相同 +spout.parallelism=3 + +#处理补全操作的bolt并行度-worker的倍数 +completion.bolt.parallelism=6 + +#写入kafka的并行度10 +kafka.bolt.parallelism=6 + +#数据中心（UID） +data.center.id.num=13 + +#hbase 更新时间，如填写0则不更新缓存 +hbase.tick.tuple.freq.secs=180 + +#app_id 更新时间，如填写0则不更新缓存 +app.tick.tuple.freq.secs=0 + +#--------------------------------默认值配置------------------------------# + +#当bolt性能受限时，限制spout接收速度，理论看ack开启才有效 +topology.config.max.spout.pending=70000 + +#hbase table name +hbase.table.name=subscriber_info + +#ack设置 1启动ack 0不启动ack +topology.num.acks=1 + +#kafka批量条数 +batch.insert.num=500 + +#tick时钟频率 +topology.tick.tuple.freq.secs=5 + +#spout接收睡眠时间 +topology.spout.sleep.time=1 + +#邮件默认编码 +mail.default.charset=UTF-8 + +#是否需要补全，yes补全/no日志原样输出 +log.need.complete=yes + +#补全校验类型 0 强制类型校验；1 弱类型校验；2 不校验 +complete.check.type=1 diff --git a/parcels/roles/init/storm/templates/RADIUS-ON-OFF.j2 b/parcels/roles/init/storm/templates/RADIUS-ON-OFF.j2 new file mode 100644 index 0000000..3b4d25c --- /dev/null +++ b/parcels/roles/init/storm/templates/RADIUS-ON-OFF.j2 @@ -0,0 +1,36 @@ +#管理kafka地址 +bootstrap.servers={{ consumer_kafka_servers }} + +#kafka broker下的topic名称 +kafka.topic=RADIUS-RECORD-LOG + +#kafka消费group id +group.id=radius-onff-log-20200818-1 + +#输出kafka server +results.output.servers={{ producer_kafka_servers }} + +#输出topic +results.output.topic=RADIUS-ONFF-LOG + +#从kafka哪里开始读：earliest/latest +auto.offset.reset=latest + +#storm topology workers +topology.workers=1 + +#storm spout parallelism +spout.parallelism=3 + +#storm bolt parallelism +format.bolt.parallelism=3 + +#tick时钟频率 +topology.tick.tuple.freq.secs=5 + +topology.config.max.spout.pending=500000 + +topology.num.acks=1 + +#kafka批量条数 +batch.insert.num=500 diff --git a/parcels/roles/init/storm/templates/RADIUS-RECORD-COMPLETED-LOG.j2 b/parcels/roles/init/storm/templates/RADIUS-RECORD-COMPLETED-LOG.j2 new file mode 100644 index 0000000..fb45b4a --- /dev/null +++ b/parcels/roles/init/storm/templates/RADIUS-RECORD-COMPLETED-LOG.j2 @@ -0,0 +1,101 @@ +#--------------------------------地址配置------------------------------# + +#管理kafka地址 +input.kafka.servers={{ consumer_kafka_servers }} + +#管理输出kafka地址 +output.kafka.servers={{ producer_kafka_servers }} + +#zookeeper 地址 用于配置log_id +zookeeper.servers={{ zookeeper_servers }} + +#hbase zookeeper地址 用于连接HBase +hbase.zookeeper.servers={{ zookeeper_servers }} + +#--------------------------------HTTP/定位库------------------------------# +#定位库地址 +ip.library={{ install_path }}/topology/dat/ + +#网关的schema位置 +schema.http=http://{{ gateway_keepalive_host }}:9999/metadata/schema/v1/fields/radius_record_log + +#网关APP_ID 获取接口 +app.id.http=http://{{ gateway_keepalive_host }}:9999/open-api/appDicList + +#--------------------------------Kafka消费组信息------------------------------# + +#kafka 接收数据topic +kafka.topic=RADIUS-RECORD-LOG + +#补全数据 输出 topic +results.output.topic=RADIUS-RECORD-COMPLETED-LOG + +#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； +group.id=radius-record-log-20210124-1 + +#生产者压缩模式 none or snappy +producer.kafka.compression.type=none + +#生产者ack +producer.ack=1 + +#latest/earliest 从当前消 or 从头消费 +auto.offset.reset=latest + +#接收自kafka的消费者 client-id +consumer.client.id=consumer-radius-record + +#回写给kafka的生产者 client-id +producer.client.id=producer-radius-record + +#--------------------------------topology配置------------------------------# + +#storm topology workers +topology.workers=1 + +#spout并行度 建议与kafka分区数相同 +spout.parallelism=3 + +#处理补全操作的bolt并行度-worker的倍数 +completion.bolt.parallelism=6 + +#写入kafka的并行度10 +kafka.bolt.parallelism=6 + +#数据中心（UID） +data.center.id.num=12 + +#hbase 更新时间，如填写0则不更新缓存 +hbase.tick.tuple.freq.secs=0 + +#app_id 更新时间，如填写0则不更新缓存 +app.tick.tuple.freq.secs=0 + +#--------------------------------默认值配置------------------------------# + +#当bolt性能受限时，限制spout接收速度，理论看ack开启才有效 +topology.config.max.spout.pending=70000 + +#hbase table name +hbase.table.name=subscriber_info + +#ack设置 1启动ack 0不启动ack +topology.num.acks=1 + +#kafka批量条数 +batch.insert.num=500 + +#tick时钟频率 +topology.tick.tuple.freq.secs=5 + +#spout接收睡眠时间 +topology.spout.sleep.time=1 + +#邮件默认编码 +mail.default.charset=UTF-8 + +#是否需要补全，yes补全/no日志原样输出 +log.need.complete=yes + +#补全校验类型 0 强制类型校验；1 弱类型校验；2 不校验 +complete.check.type=1 diff --git a/parcels/roles/init/storm/templates/SECURITY-EVENT-COMPLETED-LOG.j2 b/parcels/roles/init/storm/templates/SECURITY-EVENT-COMPLETED-LOG.j2 new file mode 100644 index 0000000..a1f5a56 --- /dev/null +++ b/parcels/roles/init/storm/templates/SECURITY-EVENT-COMPLETED-LOG.j2 @@ -0,0 +1,102 @@ +#--------------------------------地址配置------------------------------# + +#管理kafka地址 +input.kafka.servers={{ consumer_kafka_servers }} + +#管理输出kafka地址 +output.kafka.servers={{ producer_kafka_servers }} + +#zookeeper 地址 用于配置log_id +zookeeper.servers={{ zookeeper_servers }} + +#hbase zookeeper地址 用于连接HBase +hbase.zookeeper.servers={{ zookeeper_servers }} + +#--------------------------------HTTP/定位库------------------------------# +#定位库地址 +ip.library={{ install_path }}/topology/dat/ + +#网关的schema位置 +schema.http=http://{{ gateway_keepalive_host }}:9999/metadata/schema/v1/fields/security_event_log + +#网关APP_ID 获取接口 +app.id.http=http://{{ gateway_keepalive_host }}:9999/open-api/appDicList + +#--------------------------------Kafka消费组信息------------------------------# + +#kafka 接收数据topic +kafka.topic=SECURITY-EVENT-LOG + +#补全数据 输出 topic +results.output.topic=SECURITY-EVENT-COMPLETED-LOG + +#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； +group.id=security-event-log-20201209-1 + +#生产者压缩模式 none or snappy +producer.kafka.compression.type=none + +#生产者ack +producer.ack=1 + +#latest/earliest 从当前消 or 从头消费 +auto.offset.reset=latest + +#接收自kafka的消费者 client-id +consumer.client.id=consumer-security-event + +#回写给kafka的生产者 client-id +producer.client.id=producer-security-event + +#--------------------------------topology配置------------------------------# + +#storm topology workers +topology.workers=3 + +#spout并行度 建议与kafka分区数相同 +spout.parallelism=3 + +#处理补全操作的bolt并行度-worker的倍数 +completion.bolt.parallelism=12 + +#写入kafka的并行度10 +kafka.bolt.parallelism=6 + +#数据中心（UID） +data.center.id.num=11 + +#hbase 更新时间，如填写0则不更新缓存 +hbase.tick.tuple.freq.secs=180 + +#app_id 更新时间，如填写0则不更新缓存 +app.tick.tuple.freq.secs=0 + +#--------------------------------默认值配置------------------------------# + +#当bolt性能受限时，限制spout接收速度，理论看ack开启才有效 +topology.config.max.spout.pending=70000 + +#hbase table name +hbase.table.name=subscriber_info + +#ack设置 1启动ack 0不启动ack +topology.num.acks=1 + +#kafka批量条数 +batch.insert.num=500 + +#tick时钟频率 +topology.tick.tuple.freq.secs=5 + +#spout接收睡眠时间 +topology.spout.sleep.time=1 + +#邮件默认编码 +mail.default.charset=UTF-8 + +#是否需要补全，yes补全/no日志原样输出 +log.need.complete=yes + +#补全校验类型 0 强制类型校验；1 弱类型校验；2 不校验 +complete.check.type=1 + diff --git a/parcels/roles/init/storm/templates/SUBSCRUBER-HBASE.j2 b/parcels/roles/init/storm/templates/SUBSCRUBER-HBASE.j2 new file mode 100644 index 0000000..80ceb9d --- /dev/null +++ b/parcels/roles/init/storm/templates/SUBSCRUBER-HBASE.j2 @@ -0,0 +1,36 @@ +#管理kafka地址 +bootstrap.servers={{ consumer_kafka_servers }} + +#从kafka哪里开始读：earliest/latest +auto.offset.reset=latest + +#hbase zookeeper地址 +hbase.zookeeper.servers={{ zookeeper_servers }} + +#hbase table name +hbase.table.name=subscriber_info + +#tick时钟频率 +topology.tick.tuple.freq.secs=60 + +topology.config.max.spout.pending=500000 + +topology.num.acks=1 + +#kafka broker下的topic名称 +kafka.topic=RADIUS-RECORD-LOG + +#kafka消费group id +group.id=account-to-hbase-20200818-1 + +#storm topology workers +topology.workers=1 + +#storm spout parallelism +spout.parallelism=3 + +#storm bolt parallelism +format.bolt.parallelism=1 + +#spout接收睡眠时间 +topology.spout.sleep.time=200 diff --git a/parcels/roles/init/storm/templates/SYS-PACKET-CAPTURE-COMPLETED-LOG.j2 b/parcels/roles/init/storm/templates/SYS-PACKET-CAPTURE-COMPLETED-LOG.j2 new file mode 100644 index 0000000..7bfe9fb --- /dev/null +++ b/parcels/roles/init/storm/templates/SYS-PACKET-CAPTURE-COMPLETED-LOG.j2 @@ -0,0 +1,102 @@ +#--------------------------------地址配置------------------------------# + +#管理kafka地址 +input.kafka.servers={{ consumer_kafka_servers }} + +#管理输出kafka地址 +output.kafka.servers={{ producer_kafka_servers }} + +#zookeeper 地址 用于配置log_id +zookeeper.servers={{ zookeeper_servers }} + +#hbase zookeeper地址 用于连接HBase +hbase.zookeeper.servers={{ zookeeper_servers }} + +#--------------------------------HTTP/定位库------------------------------# +#定位库地址 +ip.library={{ install_path }}/topology/dat/ + +#网关的schema位置 +schema.http=http://{{ gateway_keepalive_host }}:9999/metadata/schema/v1/fields/sys_packet_capture_log + +#网关APP_ID 获取接口 +app.id.http=http://{{ gateway_keepalive_host }}:9999/open-api/appDicList + +#--------------------------------Kafka消费组信息------------------------------# + +#kafka 接收数据topic +kafka.topic=SYS-PACKET-CAPTURE-LOG + +#补全数据 输出 topic +results.output.topic=SYS-PACKET-CAPTURE-COMPLETED-LOG + +#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； +group.id=sys-packet-log-20210124-1 + +#生产者压缩模式 none or snappy +producer.kafka.compression.type=none + +#生产者ack +producer.ack=1 + +#latest/earliest 从当前消 or 从头消费 +auto.offset.reset=latest + +#接收自kafka的消费者 client-id +consumer.client.id=consumer-security-event + +#回写给kafka的生产者 client-id +producer.client.id=producer-security-event + +#--------------------------------topology配置------------------------------# + +#storm topology workers +topology.workers=1 + +#spout并行度 建议与kafka分区数相同 +spout.parallelism=3 + +#处理补全操作的bolt并行度-worker的倍数 +completion.bolt.parallelism=6 + +#写入kafka的并行度10 +kafka.bolt.parallelism=6 + +#数据中心（UID） +data.center.id.num=10 + +#hbase 更新时间，如填写0则不更新缓存 +hbase.tick.tuple.freq.secs=0 + +#app_id 更新时间，如填写0则不更新缓存 +app.tick.tuple.freq.secs=0 + +#--------------------------------默认值配置------------------------------# + +#当bolt性能受限时，限制spout接收速度，理论看ack开启才有效 +topology.config.max.spout.pending=70000 + +#hbase table name +hbase.table.name=subscriber_info + +#ack设置 1启动ack 0不启动ack +topology.num.acks=1 + +#kafka批量条数 +batch.insert.num=500 + +#tick时钟频率 +topology.tick.tuple.freq.secs=5 + +#spout接收睡眠时间 +topology.spout.sleep.time=1 + +#邮件默认编码 +mail.default.charset=UTF-8 + +#是否需要补全，yes补全/no日志原样输出 +log.need.complete=yes + +#补全校验类型 0 强制类型校验；1 弱类型校验；2 不校验 +complete.check.type=1 + diff --git a/parcels/roles/init/storm/templates/TRANSACTION-RECORD-COMPLETED-LOG.j2 b/parcels/roles/init/storm/templates/TRANSACTION-RECORD-COMPLETED-LOG.j2 new file mode 100644 index 0000000..7732bb8 --- /dev/null +++ b/parcels/roles/init/storm/templates/TRANSACTION-RECORD-COMPLETED-LOG.j2 @@ -0,0 +1,102 @@ +#--------------------------------地址配置------------------------------# + +#管理kafka地址 +input.kafka.servers={{ consumer_kafka_servers }} + +#管理输出kafka地址 +output.kafka.servers={{ producer_kafka_servers }} + +#zookeeper 地址 用于配置log_id +zookeeper.servers={{ zookeeper_servers }} + +#hbase zookeeper地址 用于连接HBase +hbase.zookeeper.servers={{ zookeeper_servers }} + +#--------------------------------HTTP/定位库------------------------------# +#定位库地址 +ip.library={{ install_path }}/topology/dat/ + +#网关的schema位置 +schema.http=http://{{ gateway_keepalive_host }}:9999/metadata/schema/v1/fields/transaction_record_log + +#网关APP_ID 获取接口 +app.id.http=http://{{ gateway_keepalive_host }}:9999/open-api/appDicList + +#--------------------------------Kafka消费组信息------------------------------# + +#kafka 接收数据topic +kafka.topic=TRANSACTION-RECORD-LOG + +#补全数据 输出 topic +results.output.topic=TRANSACTION-RECORD-COMPLETED-LOG + +#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； +group.id=transaction-record-log-20210510-A + +#生产者压缩模式 none or snappy +producer.kafka.compression.type=none + +#生产者ack +producer.ack=1 + +#latest/earliest 从当前消 or 从头消费 +auto.offset.reset=latest + +#接收自kafka的消费者 client-id +consumer.client.id=consumer-transaction-record + +#回写给kafka的生产者 client-id +producer.client.id=producer-transaction-record + +#--------------------------------topology配置------------------------------# + +#storm topology workers +topology.workers=1 + +#spout并行度 建议与kafka分区数相同 +spout.parallelism=3 + +#处理补全操作的bolt并行度-worker的倍数 +completion.bolt.parallelism=6 + +#写入kafka的并行度10 +kafka.bolt.parallelism=6 + +#数据中心（UID） +data.center.id.num=6 + +#hbase 更新时间，如填写0则不更新缓存 +hbase.tick.tuple.freq.secs=180 + +#app_id 更新时间，如填写0则不更新缓存 +app.tick.tuple.freq.secs=0 + +#--------------------------------默认值配置------------------------------# + +#当bolt性能受限时，限制spout接收速度，理论看ack开启才有效 +topology.config.max.spout.pending=70000 + +#hbase table name +hbase.table.name=subscriber_info + +#ack设置 1启动ack 0不启动ack +topology.num.acks=1 + +#kafka批量条数 +batch.insert.num=500 + +#tick时钟频率 +topology.tick.tuple.freq.secs=5 + +#spout接收睡眠时间 +topology.spout.sleep.time=1 + +#邮件默认编码 +mail.default.charset=UTF-8 + +#是否需要补全，yes补全/no日志原样输出 +log.need.complete=yes + +#补全校验类型 0 强制类型校验；1 弱类型校验；2 不校验 +complete.check.type=1 + diff --git a/parcels/roles/init/storm/templates/VOIP-RECORD-COMPLETED-LOG.j2 b/parcels/roles/init/storm/templates/VOIP-RECORD-COMPLETED-LOG.j2 new file mode 100644 index 0000000..70cfae2 --- /dev/null +++ b/parcels/roles/init/storm/templates/VOIP-RECORD-COMPLETED-LOG.j2 @@ -0,0 +1,101 @@ +#--------------------------------地址配置------------------------------# + +#管理kafka地址 +input.kafka.servers={{ consumer_kafka_servers }} + +#管理输出kafka地址 +output.kafka.servers={{ producer_kafka_servers }} + +#zookeeper 地址 用于配置log_id +zookeeper.servers={{ zookeeper_servers }} + +#hbase zookeeper地址 用于连接HBase +hbase.zookeeper.servers={{ zookeeper_servers }} + +#--------------------------------HTTP/定位库------------------------------# +#定位库地址 +ip.library={{ install_path }}/topology/dat/ + +#网关的schema位置 +schema.http=http://{{ gateway_keepalive_host }}:9999/metadata/schema/v1/fields/voip_record_log + +#网关APP_ID 获取接口 +app.id.http=http://{{ gateway_keepalive_host }}:9999/open-api/appDicList + +#--------------------------------Kafka消费组信息------------------------------# + +#kafka 接收数据topic +kafka.topic=VOIP-CONVERSATION-RECORD-LOG + +#补全数据 输出 topic +results.output.topic=VOIP-RECORD-COMPLETED-LOG + +#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； +group.id=voip-record-log-20210124-1 + +#生产者压缩模式 none or snappy +producer.kafka.compression.type=none + +#生产者ack +producer.ack=1 + +#latest/earliest 从当前消 or 从头消费 +auto.offset.reset=latest + +#接收自kafka的消费者 client-id +consumer.client.id=consumer-voip-record + +#回写给kafka的生产者 client-id +producer.client.id=producer-voip-record + +#--------------------------------topology配置------------------------------# + +#storm topology workers +topology.workers=1 + +#spout并行度 建议与kafka分区数相同 +spout.parallelism=3 + +#处理补全操作的bolt并行度-worker的倍数 +completion.bolt.parallelism=6 + +#写入kafka的并行度10 +kafka.bolt.parallelism=6 + +#数据中心（UID） +data.center.id.num=8 + +#hbase 更新时间，如填写0则不更新缓存 +hbase.tick.tuple.freq.secs=180 + +#app_id 更新时间，如填写0则不更新缓存 +app.tick.tuple.freq.secs=0 + +#--------------------------------默认值配置------------------------------# + +#当bolt性能受限时，限制spout接收速度，理论看ack开启才有效 +topology.config.max.spout.pending=70000 + +#hbase table name +hbase.table.name=subscriber_info + +#ack设置 1启动ack 0不启动ack +topology.num.acks=1 + +#kafka批量条数 +batch.insert.num=500 + +#tick时钟频率 +topology.tick.tuple.freq.secs=5 + +#spout接收睡眠时间 +topology.spout.sleep.time=1 + +#邮件默认编码 +mail.default.charset=UTF-8 + +#是否需要补全，yes补全/no日志原样输出 +log.need.complete=yes + +#补全校验类型 0 强制类型校验；1 弱类型校验；2 不校验 +complete.check.type=1 diff --git a/parcels/roles/init/storm/templates/VOIP-RELATION.j2 b/parcels/roles/init/storm/templates/VOIP-RELATION.j2 new file mode 100644 index 0000000..c198cad --- /dev/null +++ b/parcels/roles/init/storm/templates/VOIP-RELATION.j2 @@ -0,0 +1,83 @@ +#--------------------------------地址配置------------------------------# + +#管理kafka地址 +input.kafka.servers={{ consumer_kafka_servers }} + +#管理输出kafka地址 +output.kafka.servers={{ producer_kafka_servers }} + +#网关的schema位置 +schema.http=http://{{ gateway_keepalive_host }}:9999/metadata/schema/v1/fields/voip_record_log + +#kafka broker下的topic名称 +kafka.topic=VOIP-RECORD-LOG + +#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据； +group.id=voip-relation-20210407-1 + +#接收自kafka的消费者 client-id +consumer.client.id=consumer-voip-record +#回写给kafka的生产者 client-id +producer.client.id=producer-voip-record + +#生产者压缩模式 none or snappy +producer.kafka.compression.type=none + +#生产者ack +producer.ack=1 + +#latest/earliest 从当前消 or 从头消费 +auto.offset.reset=latest + +#输出topic +results.output.topic=VOIP-CONVERSATION-RECORD-LOG + +#--------------------------------topology配置------------------------------# + +#storm topology workers +topology.workers=1 + +#spout并行度 建议与kafka分区数相同 +spout.parallelism=3 + +#关联SIP的bolt并行度-关联SIP后将按照四元组拆分缓存,定时按Key分发到下一个bolt,内网IP定期缓存发入下一bolt,经下一bolt入Kafka +sip.correlation.bolt.parallelism=1 + +#关联SIP与RTP的bolt并行度-worker的倍数-关联不上的SIP或者RTP(包含四元组,SchemaType为SIP或RTP)按四元组分发到Kafka-bolt缓存,为后续二次关联做准备 +sr.correlation.bolt.parallelism=3 + +#写入kafka的并行度,同时负责二次关联,从上游拿到已关联的(VOIP)缓存后就直接写入Kafka,未关联的(SIP或RTP)就和缓存关联,关联上就放入发送Kafka的缓存,关联不上就放入未关联缓存 +kafka.bolt.parallelism=3 + +#ack设置 1启动ack 0不启动ack +topology.num.acks=1 + +#kafka批量条数 +batch.insert.num=500 + +#tick时钟频率-SIP关联缓存周期 +sip.cr.topology.tick.tuple.freq.secs=60 + +#tick时钟频率-SIP-RTP关联缓存周期-暂不使用 +sr.cr.topology.tick.tuple.freq.secs=60 + +#tick时钟频率-kafka写入周期 +topology.tick.tuple.freq.secs=5 + + +#--------------------------------默认值配置------------------------------# + +#当bolt性能受限时，限制spout接收速度，理论看ack开启才有效 +topology.config.max.spout.pending=70000 + +#spout接收睡眠时间 +topology.spout.sleep.time=1 + +#允许发送kafka最大失败数 +max.failure.num=20 + + +#--------------------------------voip------------------------------# + +#voip二次关联缓存时间-秒 +sec.combine.sr.cache.secs=300 diff --git a/parcels/roles/services/galaxy-chproxy/tasks/load_chproxy.yml b/parcels/roles/services/galaxy-chproxy/tasks/load_chproxy.yml new file mode 100644 index 0000000..83b2501 --- /dev/null +++ b/parcels/roles/services/galaxy-chproxy/tasks/load_chproxy.yml @@ -0,0 +1,131 @@ +- name: stop and remove {{ chproxy_image_container_name }} container + docker_container: + name: '{{ chproxy_image_container_name }}' + state: absent + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + +- name: remove old {{ chproxy_image_name }} image + docker_image: + name: '{{ chproxy_image_name }}' + tag: '{{ chproxy_image_tag_name }}' + state: absent + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + +- name: create {{ chproxy_volume_path }} + file: + state: directory + path: '{{ install_path }}/{{ chproxy_volume_path }}/config' + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + + +- name: create {{ install_path }}/{{ chproxy_soft_home_path }} + file: + state: directory + path: '{{ install_path }}/{{ chproxy_soft_home_path }}' + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + +- name: get allowed_host + shell: echo "{{ groups.clickhouse[0] }}" | grep -E -o "[0-9]+\.[0-9]+\.[0-9]+" + register: allowed_host_out + run_once: true + delegate_facts: true + delegate_to: "127.0.0.1" + +- name: allowed_host to ansible variable + set_fact: allowed_host="{{allowed_host_out.stdout}}" + +- name: copy {{ chproxy_image_tar_name }} + copy: + src: '{{ package_path }}/{{ chproxy_image_tar_name }}' + dest: '{{ install_path }}/{{ chproxy_soft_home_path }}/{{ chproxy_image_tar_name }}' + force: true + backup: yes + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + + +- name: load new {{ chproxy_image_name }} image from {{ chproxy_image_tar_name }} + docker_image: + name: '{{ chproxy_image_name }}' + tag: '{{ chproxy_image_tag_name }}' + load_path: '{{ install_path }}/{{ chproxy_soft_home_path }}/{{ chproxy_image_tar_name }}' + source: load + force_tag: yes + force_source: yes + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + + +- name: change the image tag + shell: docker tag chproxy:{{ chproxy_image_tag_name }} {{ chproxy_image_name }}:{{ chproxy_image_tag_name }} + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + + +- name: copy {{ chproxy_image_name }} docker-compose.yml + template: + src: docker-compose.yml.j2 + dest: '{{ install_path }}/{{ chproxy_soft_home_path }}/docker-compose.yml' + backup: yes + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + + +- name: copy {{ chproxy_image_name }} config.yml.j2 + template: + src: config.yml.j2 + dest: '{{ install_path }}/{{ chproxy_volume_path }}/config/config.yml' + backup: yes + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + + +- name: start {{ chproxy_image_container_name }} container + docker_compose: + project_src: '{{ install_path }}/{{ chproxy_soft_home_path }}' + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } diff --git a/parcels/roles/services/galaxy-chproxy/tasks/main.yml b/parcels/roles/services/galaxy-chproxy/tasks/main.yml new file mode 100644 index 0000000..a3ad8ae --- /dev/null +++ b/parcels/roles/services/galaxy-chproxy/tasks/main.yml @@ -0,0 +1,2 @@ +#- include: update_python3_and_docker_py.yml +- include: load_chproxy.yml diff --git a/parcels/roles/services/galaxy-chproxy/templates/config.yml.j2 b/parcels/roles/services/galaxy-chproxy/templates/config.yml.j2 new file mode 100644 index 0000000..6408abc --- /dev/null +++ b/parcels/roles/services/galaxy-chproxy/templates/config.yml.j2 @@ -0,0 +1,56 @@ +log_debug: false +server: + http: + listen_addr: ":8124" + allowed_networks: ["{{ allowed_host }}.0/24","127.0.0.1"] + read_timeout: 6h + idle_timeout: 10m +users: + - name: "tsg_report" + to_cluster: "report" + to_user: "tsg_report" + password: "ceiec2019" + max_queue_size: 100 + max_queue_time: 21700s + + - name: "tsg_query" + to_cluster: "query" + to_user: "tsg_query" + password: "ceiec2018" + max_queue_size: 100 + max_queue_time: 610s + cache: "shortterm" + + - name: "default" + to_cluster: "report" + to_user: "default" + password: "ceiec2019" + max_queue_size: 100 + max_queue_time: 3610s +# by default each cluster has `default` user which can be overridden by section `users` +clusters: + - name: "report" + nodes: [ + "{{groups.clickhouse[0]}}:8123", + "{{groups.clickhouse[1]}}:8123" + ] + masternode: 1 + users: + - name: "default" + password: "ceiec2019" + - name: "tsg_report" + password: "ceiec2019" + - name: "query" + nodes: [ + "{{groups.clickhouse[1]}}:8123", + "{{groups.clickhouse[0]}}:8123" + ] + masternode: 1 + users: + - name: "tsg_query" + password: "ceiec2018" +caches: + - name: "shortterm" + dir: "/home/cache" + max_size: 10000Mb + expire: 3600s diff --git a/parcels/roles/services/galaxy-chproxy/templates/docker-compose.yml.j2 b/parcels/roles/services/galaxy-chproxy/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..a71d0b3 --- /dev/null +++ b/parcels/roles/services/galaxy-chproxy/templates/docker-compose.yml.j2 @@ -0,0 +1,15 @@ +version: '3' + +services: + chproxy: + image: {{ chproxy_image_name }}:{{ chproxy_image_tag_name }} + container_name: {{ chproxy_image_container_name }} + ports: + - "8124:8124" + volumes: + - "{{ install_path }}/{{ chproxy_volume_path }}/config:/home/config" + - "{{ install_path }}/{{ chproxy_volume_path }}/log:/home/log" + - "/etc/localtime:/etc/localtime:ro" + - "/etc/timezone:/etc/timezone:ro" + restart: always + network_mode: "host" diff --git a/parcels/roles/services/galaxy-chproxy/vars/main.yml b/parcels/roles/services/galaxy-chproxy/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/services/galaxy-chproxy/vars/main.yml diff --git a/parcels/roles/services/galaxy-gateway-keepalive/tasks/main.yml b/parcels/roles/services/galaxy-gateway-keepalive/tasks/main.yml new file mode 100644 index 0000000..6722f8c --- /dev/null +++ b/parcels/roles/services/galaxy-gateway-keepalive/tasks/main.yml @@ -0,0 +1,108 @@ +#创建文件夹 +- name: create keepalived package path:{{ keepalived_package_path }} + file: + state: directory + path: '{{ keepalived_package_path }}' + +#拷贝keepalive安装包 +- name: copy keepalived-1.4.0.tar.gz + copy: + src: '{{ package_path }}/keepalived-1.4.0.tar.gz' + dest: '{{ keepalived_package_path }}/' + force: true + backup: yes + +#拷贝keepalive基础环境包 +- name: copy keepalived-depend-rpm.zip + unarchive: + src: '{{ package_path }}/keepalived-depend-rpm.zip' + dest: '{{ keepalived_package_path }}/' +# copy: yes + register: copy_info + +#安装keepalive环境包 +- name: rpm install keepalived-depend-rpm + shell: rpm -ivh {{ keepalived_package_path }}/keepalived-depend-rpm/*.rpm --force --nodeps + when: copy_info.changed + +#- name: get master ip +# shell: echo "{{ ansible_play_hosts }}" | grep -E -o "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+" | head -n 1 +# register: master_out +# run_once: true +# delegate_to: 127.0.0.1 + +- name: master_ip to ansible variable + set_fact: master_ip="{{groups.services[0]}}" + +#- name: get backup ip +# shell: echo "{{ ansible_play_hosts }}" | grep -E -o "[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+" | tail -n 1 +# register: backup_out +# run_once: true +# delegate_to: 127.0.0.1 + +- name: backup_ip to ansible variable + set_fact: backup_ip="{{groups.services[1]}}" + +#拷贝keepalive配置文件 +- name: copy app-keepalived.conf.master.j2 + template: + src: 'app-keepalived.conf.master.j2' + dest: '{{ keepalived_package_path }}/keepalived.conf' + backup: yes + run_once: true + delegate_to: "{{ master_ip }}" + + #拷贝keepalive配置文件 +- name: copy app-keepalived.conf.backup.j2 + template: + src: 'app-keepalived.conf.backup.j2' + dest: '{{ keepalived_package_path }}/keepalived.conf' + backup: yes + run_once: true + delegate_to: "{{ backup_ip }}" + +- name: copy kp_daemon.sh + template: src=kp_daemon.sh.j2 dest={{ keepalived_package_path }}/kp_daemon.sh backup=yes mode=u+x + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ master_ip }}' } + - { ip: '{{ backup_ip }}' } + + +- name: copy chk_gateway.sh + template: src=check_gateway.sh.j2 dest={{ keepalived_package_path }}/chk_gateway.sh backup=yes mode=u+x + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ master_ip }}' } + - { ip: '{{ backup_ip }}' } + +- name: copy installKeepAlived.sh + template: src=installKeepAlived.sh.j2 dest={{ keepalived_package_path }}/installKeepAlived.sh backup=yes mode=u+x + register: copy_sh_info + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ master_ip }}' } + - { ip: '{{ backup_ip }}' } + + +- name: exec installKeepAlived.sh + shell: cd {{ keepalived_package_path }} && ./installKeepAlived.sh + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ master_ip }}' } + - { ip: '{{ backup_ip }}' } + when: copy_sh_info.changed + + +- name: Ansible delete {{ keepalived_package_path }} + file: + path: "{{ keepalived_package_path }}" + state: absent diff --git a/parcels/roles/services/galaxy-gateway-keepalive/templates/app-keepalived.conf.backup.j2 b/parcels/roles/services/galaxy-gateway-keepalive/templates/app-keepalived.conf.backup.j2 new file mode 100644 index 0000000..073761b --- /dev/null +++ b/parcels/roles/services/galaxy-gateway-keepalive/templates/app-keepalived.conf.backup.j2 @@ -0,0 +1,48 @@ +! Configuration File for keepalived + + +#其实配置文件中主要要修改的选项没有很多，以下几个参数要注意 +#interface 网卡名称 +#priority 150 #优先级，同一个实例下，MASTER高于BACKUP +####下面这两项已经修改好了,建议不要修改了 +#route_id XXX #MASTER和BACKUP不同 +#virtual_router_id 51 #同一个实例下，MASTER和BACKUP相同 + +#监控服务.NGINX mysql等 +vrrp_script chk_gateway { + script "/etc/keepalived/check_gateway.sh" + #每2s检查一次 + interval 2 + #每次检查-20 + weight -20 +} + +#VRRP实例定义块 +vrrp_instance VI_1 { +#状态只有MASTER和BACKUP两种，并且要大写，MASTER为工作状态，BACKUP是备用状态。 +state BACKUP +#网卡名称 +interface {{ gateway_keepalive_interface }} +#虚拟路由标识，同一个vrrp_instance的MASTER和BACKUP的vitrual_router_id 是一致的。 +virtual_router_id {{ gateway_keepalive_router_id }} +#优先级，同一个vrrp_instance的MASTER优先级必须比BACKUP高。 +priority 100 +#MASTER 与BACKUP 负载均衡器之间同步检查的时间间隔，单位为秒。 +advert_int 1 +authentication { +#验证authentication。包含验证类型和验证密码。类型主要有PASS、AH 两种，通常使用的类型为PASS +auth_type PASS +#据说AH 使用时有问题。验证密码为明文，同一vrrp 实例MASTER 与BACKUP 使用相同的密码才能正常通信 +auth_pass 1111 +} +#触发的脚本 +track_script { +#检测脚本，上面配置的 + chk_gateway +} +virtual_ipaddress { +#虚拟ip地址,可以有多个地址，每个地址占一行，不需要子网掩码 + {{ gateway_keepalive_host }} +} +} + diff --git a/parcels/roles/services/galaxy-gateway-keepalive/templates/app-keepalived.conf.master.j2 b/parcels/roles/services/galaxy-gateway-keepalive/templates/app-keepalived.conf.master.j2 new file mode 100644 index 0000000..20c47da --- /dev/null +++ b/parcels/roles/services/galaxy-gateway-keepalive/templates/app-keepalived.conf.master.j2 @@ -0,0 +1,48 @@ +! Configuration File for keepalived + + +#其实配置文件中主要要修改的选项没有很多，以下几个参数要注意 +#interface 网卡名称 +#priority 150 #优先级，同一个实例下，MASTER高于BACKUP +####下面这两项已经修改好了,建议不要修改了 +#route_id XXX #MASTER和BACKUP不同 +#virtual_router_id 51 #同一个实例下，MASTER和BACKUP相同 + +#监控服务.NGINX mysql等 +vrrp_script chk_gateway { + script "/etc/keepalived/check_gateway.sh" + #每2s检查一次 + interval 2 + #每次检查-20 + weight -20 +} + +#VRRP实例定义块 +vrrp_instance VI_1 { +#状态只有MASTER和BACKUP两种，并且要大写，MASTER为工作状态，BACKUP是备用状态。 +state MASTER +#网卡名称 +interface {{ gateway_keepalive_interface }} +#虚拟路由标识，同一个vrrp_instance的MASTER和BACKUP的vitrual_router_id 是一致的。 +virtual_router_id {{ gateway_keepalive_router_id }} +#优先级，同一个vrrp_instance的MASTER优先级必须比BACKUP高。 +priority 150 +#MASTER 与BACKUP 负载均衡器之间同步检查的时间间隔，单位为秒。 +advert_int 1 +authentication { +#验证authentication。包含验证类型和验证密码。类型主要有PASS、AH 两种，通常使用的类型为PASS +auth_type PASS +#据说AH 使用时有问题。验证密码为明文，同一vrrp 实例MASTER 与BACKUP 使用相同的密码才能正常通信 +auth_pass 1111 +} +#触发的脚本 +track_script { +#检测脚本，上面配置的 + chk_gateway +} +virtual_ipaddress { +#虚拟ip地址,可以有多个地址，每个地址占一行，不需要子网掩码 + {{ gateway_keepalive_host }} +} +} + diff --git a/parcels/roles/services/galaxy-gateway-keepalive/templates/check_gateway.sh.j2 b/parcels/roles/services/galaxy-gateway-keepalive/templates/check_gateway.sh.j2 new file mode 100644 index 0000000..05a7907 --- /dev/null +++ b/parcels/roles/services/galaxy-gateway-keepalive/templates/check_gateway.sh.j2 @@ -0,0 +1,2 @@ +#!/bin/bash + diff --git a/parcels/roles/services/galaxy-gateway-keepalive/templates/installKeepAlived.sh.j2 b/parcels/roles/services/galaxy-gateway-keepalive/templates/installKeepAlived.sh.j2 new file mode 100644 index 0000000..0d98e47 --- /dev/null +++ b/parcels/roles/services/galaxy-gateway-keepalive/templates/installKeepAlived.sh.j2 @@ -0,0 +1,102 @@ +#!/bin/bash + + +#安装master还是backup +installMasterOrBackup=master +#keepalived安装包的名称 +keepalivedTarName=keepalived-1.4.0.tar.gz + +#wget -P /home/soft/keepalived/ http://www.keepalived.org/software/keepalived-1.4.0.tar.gz +#下载keepalived.tar到/home/soft/keepalived/目录下 +#守护进程名称 +keepalivedDaeonName=kp_daemon.sh +#master的keepalived.conf文件名称 +keepalivedMasterConfName=keepalived.conf +#bakcup的keepalived.conf文件名称 +#keepalivedBackupConfName=keepalived.conf +#检测nginx是否不用,不可用将nginx启动,在不可用杀死keepalived +checkShell=check_apps.sh + +#################################################################下面的写法,建议不要改动了################################################################# +#keepalived安装包的绝对路径 +keepalivedTarPath=`pwd`/$keepalivedTarName +#守护进程所在的目录,根据需要自行改动(会自动复制到keepalivedTarPath/sbin目录下,并添加到开机自启动中) +keepalivedDaeonPath=`pwd`/$keepalivedDaeonName +#将keepalived.tar解压到哪个目录下 +keepalivedBagPath=`pwd`/keepalived +checkShellPath=`pwd`/$checkShell +#判断是安装master还是backup +#if [ "$installMasterOrBackup" == "master" ];then + #keepalived.conf的文件路径 + keepalivedConfPath=`pwd`/$keepalivedMasterConfName +#else + #keepalived.conf的文件路径 +# keepalivedConfPath=`pwd`/$keepalivedBackupConfName +#fi + +#keepalived安装目录 +keepalivedInstallPath=/etc/keepalived +#keepalivedDaeonName=${keepalivedDaeonPath##*/} +#yum -y install gcc gcc-c++ +#yum -y install libnfnetlink-devel zlib zlib-devel openssl openssl-devel pcre pcre-devel + +#rm -rf $keepalivedBagPath + +installKP(){ + echo "开始解压 $keepalivedTarPath 到 $keepalivedBagPath" + ##--strip-components 1 + tar -xzf $keepalivedTarPath -C $keepalivedBagPath --strip-components 1 + echo "解压 $keepalivedTarPath 到 $keepalivedBagPath成功" + #cd $keepalivedBagPath && ./configure --prefix=$keepalivedInstallPath && make && make install + cd $keepalivedBagPath && ./configure && make && make install + #创建keepalived配置文件目录 + mkdir /etc/keepalived + #拷贝配置文件到/etc/keepalived目录下 + #`\cp -f $keepalivedInstallPath/etc/keepalived/keepalived.conf /etc/keepalived/ + `\cp -f $keepalivedConfPath /etc/keepalived/keepalived.conf` + #复制keepalived脚本到/etc/init.d/ 目录(开机自启动) + `\cp -f $keepalivedBagPath/keepalived/etc/init.d/keepalived /etc/init.d/keepalived` + + + + #复制keepalived守护脚本到$keepalivedInstallPath目录下 + echo "复制keepalived守护脚本到$keepalivedInstallPath目录下" + `\cp -f $keepalivedDaeonPath $keepalivedInstallPath/$keepalivedDaeonName` + #复制keepalived监控脚本到$keepalivedInstallPath目录下 + `\cp -f $checkShellPath $keepalivedInstallPath/$checkShell` + + chmod +x $keepalivedInstallPath/$keepalivedDaeonName + echo "将$keepalivedInstallPath/$keepalivedDaeonName守护进程添加到开机自启动中" + chmod +x /etc/rc.d/rc.local + echo -e "\n#设置$keepalivedDaeonName守护脚本开机自启动" >> /etc/rc.d/rc.local + echo "nohup $keepalivedInstallPath/$keepalivedDaeonName >/dev/null 2>&1 &" >> /etc/rc.d/rc.local + echo "将$keepalivedInstallPath/$keepalivedDaeonName守护进程添加到开机自启动成功" + + echo "开始启动$keepalivedInstallPath/$keepalivedDaeonName守护进程" +# nohup $keepalivedInstallPath/$keepalivedDaeonName >/dev/null 2>&1 & + service keepalived start + rm -rf $keepalivedBagPath +} + + + + +if [ ! -d $keepalivedBagPath ]; then + echo "$keepalivedBagPath目录不存在,开始创建$keepalivedBagPath目录" + mkdir -p $keepalivedBagPath + installKP +else + count=`ls $keepalivedBagPath| wc -w` + if [ $count -ne 0 ] + then + echo "安装nginx失败!!!$keepalivedBagPath有其他文件,请将文件备份到其他目录或修改nginx安装目录" + else + installKP + fi +fi + + + + + + diff --git a/parcels/roles/services/galaxy-gateway-keepalive/templates/kp_daemon.sh.j2 b/parcels/roles/services/galaxy-gateway-keepalive/templates/kp_daemon.sh.j2 new file mode 100644 index 0000000..9cde9de --- /dev/null +++ b/parcels/roles/services/galaxy-gateway-keepalive/templates/kp_daemon.sh.j2 @@ -0,0 +1,25 @@ +#!/bin/bash + +keepalivedPath=/etc/keepalived +pidFile=/var/run/keepalived.pid +logPath=$keepalivedPath/keepalived_daemon.log + +while true;do + CHPROXY_COUNT=$(ps -ef | grep chproxy | grep -v grep | wc -l) + NGINX_COUNT=$(ps -ef | grep nginx | grep -v grep | wc -l) + MARIADB_COUNT=$(ps -ef | grep mariadb | grep -v grep | wc -l) + + if [[ $MARIADB_COUNT -eq "0" || $CHPROXY_COUNT -eq "0" || $NGINX_COUNT -eq "0" ]] + then + echo "时间：`date '+%Y-%m-%d %T'` APP服务进程不存在,关闭keepalived进程" >> $logPath + systemctl stop keepalived + elif [ ! -f $pidFile ]; then + echo "时间：`date '+%Y-%m-%d %T'` keepalived进程不存在,如果服务都健康,开始启动keepalived" >> $logPath + if [[ $MARIADB_COUNT -gt "0" && $CHPROXY_COUNT -gt "0" && $NGINX_COUNT -gt "0" ]] #haproxy和nginx都不存在停止keepalived + then + systemctl start keepalived + fi + fi + sleep 5 +done + diff --git a/parcels/roles/services/galaxy-gateway-keepalive/vars/main.yml b/parcels/roles/services/galaxy-gateway-keepalive/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/services/galaxy-gateway-keepalive/vars/main.yml diff --git a/parcels/roles/services/galaxy-gateway-nginx/tasks/main.yml b/parcels/roles/services/galaxy-gateway-nginx/tasks/main.yml new file mode 100644 index 0000000..5d321d7 --- /dev/null +++ b/parcels/roles/services/galaxy-gateway-nginx/tasks/main.yml @@ -0,0 +1,132 @@ +- name: stop and remove {{ gateway_image_container_name }} container + docker_container: + name: '{{ gateway_image_container_name }}' + state: absent + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + +- name: remove old {{ gateway_image_name }} image + docker_image: + name: '{{ gateway_image_name }}' + tag: '{{ gateway_image_tag_name }}' + state: absent + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + + +- name: create {{ gateway_nginx_volume_path }} + file: + state: directory + path: '{{ install_path }}/{{ gateway_nginx_volume_path }}' + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + +- name: create {{ install_path }}/{{ gateway_nginx_soft_home_path }} + file: + state: directory + path: '{{ install_path }}/{{ gateway_nginx_soft_home_path }}' + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + +- name: copy {{ gateway_image_tar_name }} + copy: + src: '{{ package_path }}/{{ gateway_image_tar_name }}' + dest: '{{ install_path }}/{{ gateway_nginx_soft_home_path }}/' + force: true + backup: yes + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + +- name: load new {{ gateway_image_name }} image from {{ gateway_image_tar_name }} + docker_image: + name: '{{ gateway_image_name }}' + tag: '{{ gateway_image_tag_name }}' + load_path: '{{ install_path }}/{{ gateway_nginx_soft_home_path }}/{{ gateway_image_tar_name }}' + source: load + force_tag: yes + force_source: yes + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + +- name: change the image tag + shell: docker tag nginx-metrics:{{ gateway_image_tag_name }} {{ gateway_image_name }}:{{ gateway_image_tag_name }} + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + + +- name: copy {{ gateway_image_name }} docker-compose.yml + template: + src: docker-compose.yml.j2 + dest: '{{ install_path }}/{{ gateway_nginx_soft_home_path }}/docker-compose.yml' + backup: yes + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + +- name: copy {{ gateway_image_name }} nginx.conf + template: + src: nginx.conf.j2 + dest: '{{ install_path }}/{{ gateway_nginx_volume_path }}/nginx.conf' + backup: yes + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + when: (hos_keepalive_need) == "yes" + +- name: copy {{ gateway_image_name }} nginx-hos.conf + template: + src: nginx.conf-hos.j2 + dest: '{{ install_path }}/{{ gateway_nginx_volume_path }}/nginx.conf' + backup: yes + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + when: (hos_keepalive_need) == "no" + +- name: start {{ gateway_image_container_name }} container + docker_compose: + project_src: '{{ install_path }}/{{ gateway_nginx_soft_home_path }}' + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.services[0] }}' } + - { ip: '{{ groups.services[1] }}' } + diff --git a/parcels/roles/services/galaxy-gateway-nginx/templates/2106/nginx.conf-hos.j2 b/parcels/roles/services/galaxy-gateway-nginx/templates/2106/nginx.conf-hos.j2 new file mode 100644 index 0000000..b4e8d1c --- /dev/null +++ b/parcels/roles/services/galaxy-gateway-nginx/templates/2106/nginx.conf-hos.j2 @@ -0,0 +1,131 @@ +worker_processes 4; + +events { + worker_connections 1024; +} + +http { + + include mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + vhost_traffic_status_zone; + vhost_traffic_status_filter_by_host on; + + upstream qgwService { +{% for dev_info in ansible_play_hosts %} + server {{dev_info}}:8183; +{% endfor %} + } + + upstream jobAdmin { +{% for dev_info in ansible_play_hosts %} + server {{dev_info}}:8184; +{% endfor %} + } + + upstream druidQuery { + server {{ groups.druid[0] }}:8088; + server {{ groups.druid[1] }}:8088; + } + + upstream nacos { +{% for dev_info in groups.nacos %} + server {{dev_info}}:8847; +{% endfor %} + } + + server { + listen 9999; + server_name localhost; + + location / { + + proxy_pass http://qgwService; #请求转发到查询引擎集群 + + proxy_http_version 1.1; #指定使用http1.1版本 + + proxy_read_timeout 21600; #等待后端服务响应的最大时长 + + gzip on; #开启压缩 + + gzip_comp_level 6; #压缩级别 + + gzip_min_length 1k; #启用gzip压缩的最小文件,小于设置值的文件将不会压缩 + + gzip_types application/json; #压缩文件类型 + + gzip_vary on; #是否传输gzip压缩标志 + + } + } + + server { + listen 8181; + server_name localhost; + location / { + proxy_pass http://jobAdmin; + } + } + + server { + listen 8089; + server_name localhost; + location / { + proxy_pass http://druidQuery; + } + } + + server { + listen 8848; + server_name localhost; + location / { + proxy_pass http://nacos; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header REMOTE-HOST $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + } + + server { + listen 9913; + server_name localhost; + location /status { + vhost_traffic_status_display; + vhost_traffic_status_display_format html; + } + + } +} + +stream { + + # preread_buffer_size 64k; + # preread_timeout 90s; + # proxy_protocol_timeout 300s; + # resolver_timeout 10s; + + log_format proxy '$remote_addr [$time_local] ' + '$protocol $status $bytes_sent $bytes_received ' + '$session_time "$upstream_addr" ' + '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"'; + + access_log /usr/local/nginx/logs/access.log proxy ; + open_log_file_cache off; + include /etc/nginx/conf.d/*.stream; + + upstream hos { +{% for dev_info in groups.hos %} + server {{dev_info}}:8186 max_fails=1 fail_timeout=60s; +{% endfor %} + } + + server { + listen 9098; + proxy_connect_timeout 20s; + proxy_timeout 20s; + proxy_pass hos; + } +} + diff --git a/parcels/roles/services/galaxy-gateway-nginx/templates/2106/nginx.conf.j2 b/parcels/roles/services/galaxy-gateway-nginx/templates/2106/nginx.conf.j2 new file mode 100644 index 0000000..ef5461b --- /dev/null +++ b/parcels/roles/services/galaxy-gateway-nginx/templates/2106/nginx.conf.j2 @@ -0,0 +1,101 @@ +worker_processes 4; + +events { + worker_connections 1024; +} + +http { + + include mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + vhost_traffic_status_zone; + vhost_traffic_status_filter_by_host on; + + upstream qgwService { +{% for dev_info in ansible_play_hosts %} + server {{dev_info}}:8183; +{% endfor %} + } + + upstream jobAdmin { +{% for dev_info in ansible_play_hosts %} + server {{dev_info}}:8184; +{% endfor %} + } + + upstream druidQuery { + server {{ groups.druid[0] }}:8088; + server {{ groups.druid[1] }}:8088; + } + + upstream nacos { +{% for dev_info in groups.nacos %} + server {{dev_info}}:8847; +{% endfor %} + } + + server { + listen 9999; + server_name localhost; + + location / { + + proxy_pass http://qgwService; #请求转发到查询引擎集群 + + proxy_http_version 1.1; #指定使用http1.1版本 + + proxy_read_timeout 21600; #等待后端服务响应的最大时长 + + gzip on; #开启压缩 + + gzip_comp_level 6; #压缩级别 + + gzip_min_length 1k; #启用gzip压缩的最小文件,小于设置值的文件将不会压缩 + + gzip_types application/json; #压缩文件类型 + + gzip_vary on; #是否传输gzip压缩标志 + + } + } + + server { + listen 8181; + server_name localhost; + location / { + proxy_pass http://jobAdmin; + } + } + + server { + listen 8089; + server_name localhost; + location / { + proxy_pass http://druidQuery; + } + } + + server { + listen 8848; + server_name localhost; + location / { + proxy_pass http://nacos; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header REMOTE-HOST $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + } + + server { + listen 9913; + server_name localhost; + location /status { + vhost_traffic_status_display; + vhost_traffic_status_display_format html; + } + + } + +} diff --git a/parcels/roles/services/galaxy-gateway-nginx/templates/docker-compose.yml.j2 b/parcels/roles/services/galaxy-gateway-nginx/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..127f3f8 --- /dev/null +++ b/parcels/roles/services/galaxy-gateway-nginx/templates/docker-compose.yml.j2 @@ -0,0 +1,14 @@ +version: '3' +services: + nginx: + image: {{ gateway_image_name }}:{{ gateway_image_tag_name }} + container_name: {{ gateway_image_container_name }} + restart: always + ports: + - 80:80 + volumes: + - {{ install_path }}/{{ gateway_nginx_volume_path }}/logs:/usr/local/nginx/logs + - {{ install_path }}/{{ gateway_nginx_volume_path }}/nginx.conf:/usr/local/nginx/conf/nginx.conf + working_dir: /etc/nginx + command: /etc/nginx/nginx -g 'daemon off;' + network_mode: "host" diff --git a/parcels/roles/services/galaxy-gateway-nginx/templates/nginx.conf-hos.j2 b/parcels/roles/services/galaxy-gateway-nginx/templates/nginx.conf-hos.j2 new file mode 100644 index 0000000..b4e8d1c --- /dev/null +++ b/parcels/roles/services/galaxy-gateway-nginx/templates/nginx.conf-hos.j2 @@ -0,0 +1,131 @@ +worker_processes 4; + +events { + worker_connections 1024; +} + +http { + + include mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + vhost_traffic_status_zone; + vhost_traffic_status_filter_by_host on; + + upstream qgwService { +{% for dev_info in ansible_play_hosts %} + server {{dev_info}}:8183; +{% endfor %} + } + + upstream jobAdmin { +{% for dev_info in ansible_play_hosts %} + server {{dev_info}}:8184; +{% endfor %} + } + + upstream druidQuery { + server {{ groups.druid[0] }}:8088; + server {{ groups.druid[1] }}:8088; + } + + upstream nacos { +{% for dev_info in groups.nacos %} + server {{dev_info}}:8847; +{% endfor %} + } + + server { + listen 9999; + server_name localhost; + + location / { + + proxy_pass http://qgwService; #请求转发到查询引擎集群 + + proxy_http_version 1.1; #指定使用http1.1版本 + + proxy_read_timeout 21600; #等待后端服务响应的最大时长 + + gzip on; #开启压缩 + + gzip_comp_level 6; #压缩级别 + + gzip_min_length 1k; #启用gzip压缩的最小文件,小于设置值的文件将不会压缩 + + gzip_types application/json; #压缩文件类型 + + gzip_vary on; #是否传输gzip压缩标志 + + } + } + + server { + listen 8181; + server_name localhost; + location / { + proxy_pass http://jobAdmin; + } + } + + server { + listen 8089; + server_name localhost; + location / { + proxy_pass http://druidQuery; + } + } + + server { + listen 8848; + server_name localhost; + location / { + proxy_pass http://nacos; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header REMOTE-HOST $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + } + + server { + listen 9913; + server_name localhost; + location /status { + vhost_traffic_status_display; + vhost_traffic_status_display_format html; + } + + } +} + +stream { + + # preread_buffer_size 64k; + # preread_timeout 90s; + # proxy_protocol_timeout 300s; + # resolver_timeout 10s; + + log_format proxy '$remote_addr [$time_local] ' + '$protocol $status $bytes_sent $bytes_received ' + '$session_time "$upstream_addr" ' + '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"'; + + access_log /usr/local/nginx/logs/access.log proxy ; + open_log_file_cache off; + include /etc/nginx/conf.d/*.stream; + + upstream hos { +{% for dev_info in groups.hos %} + server {{dev_info}}:8186 max_fails=1 fail_timeout=60s; +{% endfor %} + } + + server { + listen 9098; + proxy_connect_timeout 20s; + proxy_timeout 20s; + proxy_pass hos; + } +} + diff --git a/parcels/roles/services/galaxy-gateway-nginx/templates/nginx.conf.j2 b/parcels/roles/services/galaxy-gateway-nginx/templates/nginx.conf.j2 new file mode 100644 index 0000000..0578418 --- /dev/null +++ b/parcels/roles/services/galaxy-gateway-nginx/templates/nginx.conf.j2 @@ -0,0 +1,109 @@ +worker_processes 4; + +events { + worker_connections 1024; +} + +http { + include mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + vhost_traffic_status_zone; + vhost_traffic_status_filter_by_host on; + + upstream qgwService { +{% for dev_info in ansible_play_hosts %} + server {{dev_info}}:8183; +{% endfor %} + } + + upstream jobAdmin { +{% for dev_info in ansible_play_hosts %} + server {{dev_info}}:8184; +{% endfor %} + } + + upstream druidQuery { + server {{ groups.druid[0] }}:8088; + server {{ groups.druid[1] }}:8088; + } + + server { + listen 9999; + server_name localhost; + + location / { + proxy_pass http://qgwService; #请求转发到查询引擎集群 + proxy_http_version 1.1; #指定使用http1.1版本 + proxy_read_timeout 21600; #等待后端服务响应的最大时长 + gzip on; #开启压缩 + gzip_comp_level 6; #压缩级别 + gzip_min_length 1k; #启用gzip压缩的最小文件,小于设置值的文件将不会压缩 + gzip_types application/json; #压缩文件类型 + gzip_vary on; #是否传输gzip压缩标志 + } + } + + server { + listen 8181; + server_name localhost; + location / { + proxy_pass http://jobAdmin; + } + } + + server { + listen 8089; + server_name localhost; + location / { + proxy_pass http://druidQuery; + } + } + + server { + listen 9913; + server_name localhost; + location /status { + vhost_traffic_status_display; + vhost_traffic_status_display_format html; + } + } + +} + +stream { + + upstream nacos { +{% for dev_info in groups.nacos %} + server {{dev_info}}:8847 max_fails=2 fail_timeout=10s; +{% endfor %} + } + upstream nacosGrpc { +{% for dev_info in groups.nacos %} + server {{dev_info}}:9847 max_fails=2 fail_timeout=10s; +{% endfor %} + } + + server { + listen 8848; + proxy_pass nacos; + proxy_timeout 120; #等待后端服务响应的最大时长 + proxy_connect_timeout 30s; #与被代理服务器建立连接的超时时间为5s + proxy_next_upstream on; # 当被代理的服务器返回错误或超时时，将未返回响应的客户端连接请求传递给upstream中的下一个服务器 + proxy_next_upstream_tries 3; # 转发尝试请求最多3次 + proxy_next_upstream_timeout 10s; # 总尝试超时时间为10s + proxy_socket_keepalive on; # 开启SO_KEEPALIVE选项进行心跳检测 + } + server { + listen 9848; + proxy_pass nacosGrpc; + proxy_timeout 120; + proxy_connect_timeout 30s; + proxy_next_upstream on; + proxy_next_upstream_tries 3; + proxy_next_upstream_timeout 10s; + proxy_socket_keepalive on; + } +} + diff --git a/parcels/roles/services/galaxy-gateway-nginx/vars/main.yml b/parcels/roles/services/galaxy-gateway-nginx/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/services/galaxy-gateway-nginx/vars/main.yml diff --git a/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_active_defence_event_log_tsgv3.sh b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_active_defence_event_log_tsgv3.sh new file mode 100755 index 0000000..88a1ca7 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_active_defence_event_log_tsgv3.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +#gohangout启动文件 + +#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下 +YML_NAME=k2ck_active_defence_event_log_tsgv3 +#gohangout的二进制启动文件路径 +BASE_DIR=$(cd $(dirname $0); cd ../../; pwd) +#日志级别,1,5,10,数字越大日志越详细 +LOG_LV=3 +#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程 +THREAD_SUM=3 +#进程总数 +PROCESS_SUM=$1 + +if [ ! -d "$BASE_DIR/logs" ]; then + mkdir -p $BASE_DIR/logs +fi + +echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_connection_record_log_tsgv3.sh b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_connection_record_log_tsgv3.sh new file mode 100755 index 0000000..9a2fdae --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_connection_record_log_tsgv3.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +#gohangout启动文件 + +#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下 +YML_NAME=k2ck_connection_record_log_tsgv3 +#gohangout的二进制启动文件路径 +BASE_DIR=$(cd $(dirname $0); cd ../../; pwd) +#日志级别,1,5,10,数字越大日志越详细 +LOG_LV=3 +#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程 +THREAD_SUM=3 +#进程总数 +PROCESS_SUM=$1 + +if [ ! -d "$BASE_DIR/logs" ]; then + mkdir -p $BASE_DIR/logs +fi + +echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_gtpc_record_log_tsgv3.sh b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_gtpc_record_log_tsgv3.sh new file mode 100755 index 0000000..d35b7dc --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_gtpc_record_log_tsgv3.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +#gohangout启动文件 + +#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下 +YML_NAME=k2ck_gtpc_record_log_tsgv3 +#gohangout的二进制启动文件路径 +BASE_DIR=$(cd $(dirname $0); cd ../../; pwd) +#日志级别,1,5,10,数字越大日志越详细 +LOG_LV=5 +#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程 +THREAD_SUM=3 +#进程总数 +PROCESS_SUM=$1 + +if [ ! -d "$BASE_DIR/logs" ]; then + mkdir -p $BASE_DIR/logs +fi + +echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_live_session_record_log_tsgv3.sh b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_live_session_record_log_tsgv3.sh new file mode 100755 index 0000000..df02451 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_live_session_record_log_tsgv3.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +#gohangout启动文件 + +#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下 +YML_NAME=k2ck_live_session_record_log_tsgv3 +#gohangout的二进制启动文件路径 +BASE_DIR=$(cd $(dirname $0); cd ../../; pwd) +#日志级别,1,5,10,数字越大日志越详细 +LOG_LV=5 +#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程 +THREAD_SUM=3 +#进程总数 +PROCESS_SUM=$1 + +if [ ! -d "$BASE_DIR/logs" ]; then + mkdir -p $BASE_DIR/logs +fi + +echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_proxy_event_log_tsgv3.sh b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_proxy_event_log_tsgv3.sh new file mode 100755 index 0000000..8159543 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_proxy_event_log_tsgv3.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +#gohangout启动文件 + +#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下 +YML_NAME=k2ck_proxy_event_log_tsgv3 +#gohangout的二进制启动文件路径 +BASE_DIR=$(cd $(dirname $0); cd ../../; pwd) +#日志级别,1,5,10,数字越大日志越详细 +LOG_LV=3 +#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程 +THREAD_SUM=3 +#进程总数 +PROCESS_SUM=$1 + +if [ ! -d "$BASE_DIR/logs" ]; then + mkdir -p $BASE_DIR/logs +fi + +echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_radius_onff_log_tsgv3.sh b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_radius_onff_log_tsgv3.sh new file mode 100755 index 0000000..f199375 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_radius_onff_log_tsgv3.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +#gohangout启动文件 + +#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下 +YML_NAME=k2ck_radius_onff_log_tsgv3 +#gohangout的二进制启动文件路径 +BASE_DIR=$(cd $(dirname $0); cd ../../; pwd) +#日志级别,1,5,10,数字越大日志越详细 +LOG_LV=3 +#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程 +THREAD_SUM=3 +#进程总数 +PROCESS_SUM=$1 + +if [ ! -d "$BASE_DIR/logs" ]; then + mkdir -p $BASE_DIR/logs +fi + +echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_radius_record_log_tsgv3.sh b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_radius_record_log_tsgv3.sh new file mode 100755 index 0000000..c7cf977 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_radius_record_log_tsgv3.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +#gohangout启动文件 + +#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下 +YML_NAME=k2ck_radius_record_log_tsgv3 +#gohangout的二进制启动文件路径 +BASE_DIR=$(cd $(dirname $0); cd ../../; pwd) +#日志级别,1,5,10,数字越大日志越详细 +LOG_LV=3 +#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程 +THREAD_SUM=3 +#进程总数 +PROCESS_SUM=$1 + +if [ ! -d "$BASE_DIR/logs" ]; then + mkdir -p $BASE_DIR/logs +fi + +echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_security_event_log_tsgv3.sh b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_security_event_log_tsgv3.sh new file mode 100755 index 0000000..782c8da --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_security_event_log_tsgv3.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +#gohangout启动文件 + +#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下 +YML_NAME=k2ck_security_event_log_tsgv3 +#gohangout的二进制启动文件路径 +BASE_DIR=$(cd $(dirname $0); cd ../../; pwd) +#日志级别,1,5,10,数字越大日志越详细 +LOG_LV=3 +#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程 +THREAD_SUM=3 +#进程总数 +PROCESS_SUM=$1 + +if [ ! -d "$BASE_DIR/logs" ]; then + mkdir -p $BASE_DIR/logs +fi + +echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_sys_packet_capture_log_tsgv3.sh b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_sys_packet_capture_log_tsgv3.sh new file mode 100755 index 0000000..3799da9 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_sys_packet_capture_log_tsgv3.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +#gohangout启动文件 + +#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下 +YML_NAME=k2ck_sys_packet_capture_log_tsgv3 +#gohangout的二进制启动文件路径 +BASE_DIR=$(cd $(dirname $0); cd ../../; pwd) +#日志级别,1,5,10,数字越大日志越详细 +LOG_LV=3 +#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程 +THREAD_SUM=3 +#进程总数 +PROCESS_SUM=$1 + +if [ ! -d "$BASE_DIR/logs" ]; then + mkdir -p $BASE_DIR/logs +fi + +echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_transaction_record_log_tsgv3.sh b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_transaction_record_log_tsgv3.sh new file mode 100755 index 0000000..74826b4 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_transaction_record_log_tsgv3.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +#gohangout启动文件 + +#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下 +YML_NAME=k2ck_transaction_record_log_tsgv3 +#gohangout的二进制启动文件路径 +BASE_DIR=$(cd $(dirname $0); cd ../../; pwd) +#日志级别,1,5,10,数字越大日志越详细 +LOG_LV=5 +#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程 +THREAD_SUM=3 +#进程总数 +PROCESS_SUM=$1 + +if [ ! -d "$BASE_DIR/logs" ]; then + mkdir -p $BASE_DIR/logs +fi + +echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_voip_record_log_tsgv3.sh b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_voip_record_log_tsgv3.sh new file mode 100755 index 0000000..37972b7 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/files/bin/ghoStart/start_gohangout_k2ck_voip_record_log_tsgv3.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +#gohangout启动文件 + +#配置文件名称,注意配置文件需要放在$BASE_DIR/conf文件夹下 +YML_NAME=k2ck_voip_record_log_tsgv3 +#gohangout的二进制启动文件路径 +BASE_DIR=$(cd $(dirname $0); cd ../../; pwd) +#日志级别,1,5,10,数字越大日志越详细 +LOG_LV=3 +#线程总数,即开启worker数量,每个线程拥有自己的filter, output,占用多份内存,默认1个线程 +THREAD_SUM=3 +#进程总数 +PROCESS_SUM=$1 + +if [ ! -d "$BASE_DIR/logs" ]; then + mkdir -p $BASE_DIR/logs +fi + +echo "###########################$(date +%Y%m%d%H%M%S)###########################" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log +id=0 +logid=0 +while true ; do + NUM=`ps -ef | grep -w ${YML_NAME}.yml | grep -v grep |wc -l` + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + time_stamp=$(date +%Y%m%d%H%M%S) + if [ "${NUM}" -lt ${PROCESS_SUM} ];then + $BASE_DIR/bin/gohangout --config $BASE_DIR/conf/${YML_NAME}.yml --logtostderr --v ${LOG_LV} --worker ${THREAD_SUM} >> $BASE_DIR/logs/${YML_NAME}.log 2>&1 & + echo "${time_stamp} ---> the ${YML_NAME}APP restart ---> $id" >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + ((logid++)) + ((id++)) + if [ ${logid} -gt ${PROCESS_SUM} ];then + logid=0 + pids=$(ps -ef | grep -w ${YML_NAME}.yml | grep -v grep | awk '{print $2}') + for pid in $pids + do + kill -9 $pid + done + echo "kafka or clickhouse is error,reset gohangout,sleep 30s... ..." >> $BASE_DIR/bin/start_log/${YML_NAME}_restart.log + sleep 30 + fi + #大于设置进程数,杀掉所有进程,重启 + elif [ "${NUM}" -gt ${PROCESS_SUM} ];then + for pid in $pids + do + kill -9 $pid + done + id=0 + fi + sleep 1 +done diff --git a/parcels/roles/services/galaxy-gohangout/files/start_all.sh b/parcels/roles/services/galaxy-gohangout/files/start_all.sh new file mode 100755 index 0000000..81dcaf8 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/files/start_all.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +STARTDIR=$(cd $(dirname $0); pwd) + + +#docker模式 +#nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_active_defence_event_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_connection_record_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_proxy_event_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_radius_record_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_radius_onff_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_sys_packet_capture_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_voip_record_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_gtpc_record_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_live_session_record_log_tsgv3.sh $1 > /dev/null 2>&1 & +nohup $STARTDIR/bin/ghoStart/start_gohangout_k2ck_transaction_record_log_tsgv3.sh $1 > /dev/null 2>&1 & +$STARTDIR/bin/ghoStart/start_gohangout_k2ck_security_event_log_tsgv3.sh $1 + diff --git a/parcels/roles/services/galaxy-gohangout/tasks/load_gohangout.yml b/parcels/roles/services/galaxy-gohangout/tasks/load_gohangout.yml new file mode 100644 index 0000000..62d8d45 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/tasks/load_gohangout.yml @@ -0,0 +1,148 @@ +- name: stop and remove {{ gohangout_image_container_name }} container + docker_container: + name: '{{ gohangout_image_container_name }}' + state: absent + +- name: remove old {{ gohangout_image_name }} image + docker_image: + name: '{{ gohangout_image_name }}' + tag: '{{ gohangout_image_tag_name }}' + state: absent + +- name: create {{ install_path }}/{{ gohangout_soft_home_path }} + file: + state: directory + path: '{{ install_path }}/{{ gohangout_soft_home_path }}' + +- name: create gohangout volume + file: + state: directory + path: '{{ install_path }}/{{ gohangout_volume_path }}' + +- name: create gohangout conf volume + file: + state: directory + path: '{{ install_path }}/{{ gohangout_volume_path }}/conf' + +- name: copy bin file + copy: src=files/bin dest={{ install_path }}/{{ gohangout_volume_path }} force=true backup=yes + +- name: copy config file + copy: src=files/start_all.sh dest={{ install_path }}/{{ gohangout_volume_path }} force=true backup=yes + +- name: execute the shell + shell: chmod +x {{ install_path }}/{{ gohangout_volume_path }}/start_all.sh + +- name: execute the shell + shell: chmod +x {{ install_path }}/{{ gohangout_volume_path }}/bin/ghoStart/*.sh + +- name: copy {{ gohangout_image_tar_name }} + copy: + src: '{{ package_path }}/{{ gohangout_image_tar_name }}' + dest: '{{ install_path }}/{{ gohangout_soft_home_path }}/{{ gohangout_image_tar_name }}' + force: true + backup: yes + +- name: load new {{ gohangout_image_name }} image from {{ gohangout_image_tar_name }} + docker_image: + name: '{{ gohangout_image_name }}' + tag: '{{ gohangout_image_tag_name }}' + load_path: '{{ install_path }}/{{ gohangout_soft_home_path }}/{{ gohangout_image_tar_name }}' + source: load + force_tag: yes + force_source: yes + timeout: 300 + +- name: copy {{ gohangout_image_name }} docker-compose.yml + template: + src: docker-compose.yml.j2 + dest: '{{ install_path }}/{{ gohangout_soft_home_path }}/docker-compose.yml' + backup: yes + +- name: copy {{ gohangout_image_name }} k2ck_connection_record_log_tsgv3.yml + template: + src: k2ck_connection_record_log_tsgv3.yml.j2 + dest: '{{ install_path }}/{{ gohangout_volume_path }}/conf/k2ck_connection_record_log_tsgv3.yml' + backup: yes + +- name: copy {{ gohangout_image_name }} k2ck_proxy_event_log_tsgv3.yml + template: + src: k2ck_proxy_event_log_tsgv3.yml.j2 + dest: '{{ install_path }}/{{ gohangout_volume_path }}/conf/k2ck_proxy_event_log_tsgv3.yml' + backup: yes +- name: copy {{ gohangout_image_name }} k2ck_radius_onff_log_tsgv3.yml + template: + src: k2ck_radius_onff_log_tsgv3.yml.j2 + dest: '{{ install_path }}/{{ gohangout_volume_path }}/conf/k2ck_radius_onff_log_tsgv3.yml' + backup: yes + +- name: copy {{ gohangout_image_name }} k2ck_radius_record_log_tsgv3.yml + template: + src: k2ck_radius_record_log_tsgv3.yml.j2 + dest: '{{ install_path }}/{{ gohangout_volume_path }}/conf/k2ck_radius_record_log_tsgv3.yml' + backup: yes + +- name: copy {{ gohangout_image_name }} k2ck_security_event_log_tsgv3.yml + template: + src: k2ck_security_event_log_tsgv3.yml.j2 + dest: '{{ install_path }}/{{ gohangout_volume_path }}/conf/k2ck_security_event_log_tsgv3.yml' + backup: yes + +- name: copy {{ gohangout_image_name }} k2ck_sys_packet_capture_log_tsgv3.yml + template: + src: k2ck_sys_packet_capture_log_tsgv3.yml.j2 + dest: '{{ install_path }}/{{ gohangout_volume_path }}/conf/k2ck_sys_packet_capture_log_tsgv3.yml' + backup: yes + +- name: copy {{ gohangout_image_name }} k2ck_active_defence_event_log_tsgv3.yml + template: + src: k2ck_active_defence_event_log_tsgv3.yml.j2 + dest: '{{ install_path }}/{{ gohangout_volume_path }}/conf/k2ck_active_defence_event_log_tsgv3.yml' + backup: yes + +- name: copy {{ gohangout_image_name }} k2ck_voip_record_log_tsgv3.yml + template: + src: k2ck_voip_record_log_tsgv3.yml.j2 + dest: '{{ install_path }}/{{ gohangout_volume_path }}/conf/k2ck_voip_record_log_tsgv3.yml' + backup: yes + +- name: copy {{ gohangout_image_name }} k2ck_gtpc_record_log_tsgv3.yml + template: + src: k2ck_gtpc_record_log_tsgv3.yml.j2 + dest: '{{ install_path }}/{{ gohangout_volume_path }}/conf/k2ck_gtpc_record_log_tsgv3.yml' + backup: yes + +- name: copy {{ gohangout_image_name }} k2ck_live_session_record_log_tsgv3.yml + template: + src: k2ck_live_session_record_log_tsgv3.yml.j2 + dest: '{{ install_path }}/{{ gohangout_volume_path }}/conf/k2ck_live_session_record_log_tsgv3.yml' + backup: yes + +- name: copy {{ gohangout_image_name }} k2ck_transaction_record_log_tsgv3.yml + template: + src: k2ck_transaction_record_log_tsgv3.yml.j2 + dest: '{{ install_path }}/{{ gohangout_volume_path }}/conf/k2ck_transaction_record_log_tsgv3.yml' + backup: yes + +- name: copy {{ gohangout_image_name }} gholog.j2 + template: + src: gholog.j2 + dest: '/etc/logrotate.d/gholog' + backup: yes + +- name: start {{ gohangout_image_container_name }} container + docker_compose: + project_src: '{{ install_path }}/{{ gohangout_soft_home_path }}' + +#- name: wait gohangout start ,sleep 30s +# shell: sleep 30 + +#- name: check gohangout start job num +# shell: "ps -ef | grep gohangout | grep worker | grep -v grep | wc -l" +# register: check_out +# +#- fail: +# msg: "{{ inventory_hostname }}- gohangout start job num is not all. Please check it" +# when: check_out.stdout < '10' + + diff --git a/parcels/roles/services/galaxy-gohangout/tasks/main.yml b/parcels/roles/services/galaxy-gohangout/tasks/main.yml new file mode 100644 index 0000000..6b9863e --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/tasks/main.yml @@ -0,0 +1,3 @@ +#- include: update_python3_and_docker_py.yml +#- include: test.yml +- include: load_gohangout.yml diff --git a/parcels/roles/services/galaxy-gohangout/tasks/test.yml b/parcels/roles/services/galaxy-gohangout/tasks/test.yml new file mode 100644 index 0000000..d829801 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/tasks/test.yml @@ -0,0 +1,83 @@ +- name: create {{ gohangout_soft_home_path }} + file: + state: directory + path: '{{ gohangout_soft_home_path }}' + +- name: create gohangout volume + file: + state: directory + path: '{{ gohangout_volume_path }}' +- name: create gohangout conf volume + file: + state: directory + path: '{{ gohangout_volume_path }}/conf' + + +- name: copy config file + copy: src=file/bin dest={{ gohangout_volume_path }} force=true backup=yes + +- name: copy start_all.sh script + copy: src=file/start_all.sh dest={{ gohangout_volume_path }}/start_all.sh force=true backup=yes + +- name: execute the shell + shell: chmod +x {{ gohangout_volume_path }}/start_all.sh + +- name: execute the shell + shell: chmod +x {{ gohangout_volume_path }}/bin/ghoStart/*.sh + +- name: copy {{ image_tar_name }} + copy: src=file/{{ image_tar_name }} dest={{ gohangout_soft_home_path }}/{{ image_tar_name }} force=true backup=yes + when: offline_install #离线模式从本地load文件加载 + +- name: load new {{ image_name }} image from {{ image_tar_name }} + docker_image: + name: '{{ image_name }}' + tag: '{{ image_tag_name }}' + load_path: '{{ gohangout_soft_home_path }}/{{ image_tar_name }}' + source: load + force_tag: yes + force_source: yes + when: offline_install #离线模式从本地load文件加载 + +- name: load new {{ image_name }} image from docker-registry + docker_image: + name: '{{ docker_registry_image_and_tag }}' + source: pull + force_tag: yes + force_source: yes + when: offline_install==false #在线模式从docker-registry中pull镜像 + +- name: copy {{ image_name }} docker-compose.yml + template: + src: docker-compose.yml.j2 + dest: '{{ gohangout_soft_home_path }}/docker-compose.yml' + backup: yes + +- name: copy {{ image_name }} k2ck_connection_record_log_tsgv3.yml + template: + src: k2ck_connection_record_log_tsgv3.yml.j2 + dest: '{{ gohangout_volume_path }}/conf/k2ck_connection_record_log_tsgv3.yml' + backup: yes + +- name: copy {{ image_name }} k2ck_proxy_event_log_tsgv3.yml + template: + src: k2ck_proxy_event_log_tsgv3.yml.j2 + dest: '{{ gohangout_volume_path }}/conf/k2ck_proxy_event_log_tsgv3.yml' + backup: yes +- name: copy {{ image_name }} k2ck_radius_onff_log_tsgv3.yml + template: + src: k2ck_radius_onff_log_tsgv3.yml.j2 + dest: '{{ gohangout_volume_path }}/conf/k2ck_radius_onff_log_tsgv3.yml' + backup: yes + +- name: copy {{ image_name }} k2ck_radius_record_log_tsgv3.yml + template: + src: k2ck_radius_record_log_tsgv3.yml.j2 + dest: '{{ gohangout_volume_path }}/conf/k2ck_radius_record_log_tsgv3.yml' + backup: yes + +- name: copy {{ image_name }} k2ck_security_event_log_tsgv3.yml + template: + src: k2ck_security_event_log_tsgv3.yml.j2 + dest: '{{ gohangout_volume_path }}/conf/k2ck_security_event_log_tsgv3.yml' + backup: yes diff --git a/parcels/roles/services/galaxy-gohangout/templates/docker-compose.yml.j2 b/parcels/roles/services/galaxy-gohangout/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..2bdb856 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/templates/docker-compose.yml.j2 @@ -0,0 +1,17 @@ +version: "3" +services: + gohangout: + #依赖的镜像 + image: {{ gohangout_image_name }}:{{ gohangout_image_tag_name }} + restart: always + container_name: {{ gohangout_image_container_name }} + environment: + PROCESSNUM: 1 + volumes: + - "{{ install_path }}/{{ gohangout_volume_path }}/logs:/home/ceiec/go_gohangout/gohangout/logs" + - "{{ install_path }}/{{ gohangout_volume_path }}/start_all.sh:/home/ceiec/go_gohangout/gohangout/start_all.sh" + - "{{ install_path }}/{{ gohangout_volume_path }}/conf:/home/ceiec/go_gohangout/gohangout/conf" + - "{{ install_path }}/{{ gohangout_volume_path }}/bin/ghoStart:/home/ceiec/go_gohangout/gohangout/bin/ghoStart" + - "/etc/localtime:/etc/localtime:ro" + - "/etc/timezone:/etc/timezone:ro" + network_mode: "host" diff --git a/parcels/roles/services/galaxy-gohangout/templates/gholog.j2 b/parcels/roles/services/galaxy-gohangout/templates/gholog.j2 new file mode 100644 index 0000000..572b27a --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/templates/gholog.j2 @@ -0,0 +1,15 @@ +{{ install_path }}/galaxy/volumes/gohangout/logs/*.log { + size 500M + rotate 10 + missingok + notifempty + compress + delaycompress + maxage 180 + dateext + dateformat -%Y%m%d-%s + copytruncate + createolddir 0755 root root + olddir ./bak_gholog +} + diff --git a/parcels/roles/services/galaxy-gohangout/templates/k2ck_active_defence_event_log_tsgv3.yml.j2 b/parcels/roles/services/galaxy-gohangout/templates/k2ck_active_defence_event_log_tsgv3.yml.j2 new file mode 100644 index 0000000..709d82c --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/templates/k2ck_active_defence_event_log_tsgv3.yml.j2 @@ -0,0 +1,31 @@ +inputs: + - Kafka: + topic: + ACTIVE-DEFENCE-EVENT-COMPLETED-LOG: 3 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "{{ consumer_kafka_servers }}" + group.id: active_defence_event_tsgv3 + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'tsg_galaxy_v3.active_defence_event_log_local' + username: 'default' + password: 'ceiec2019' + hosts: +{% for ip in groups.clickhouse %} +{% if (groups.clickhouse[0]) != ip and (groups.clickhouse[1]) != ip%} + - 'tcp://{{ ip }}:9001' +{% endif %} +{% endfor %} + bulk_actions: 100000 + flush_interval: 30 + concurrent: 2 diff --git a/parcels/roles/services/galaxy-gohangout/templates/k2ck_connection_record_log_tsgv3.yml.j2 b/parcels/roles/services/galaxy-gohangout/templates/k2ck_connection_record_log_tsgv3.yml.j2 new file mode 100644 index 0000000..45987e1 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/templates/k2ck_connection_record_log_tsgv3.yml.j2 @@ -0,0 +1,32 @@ +inputs: + - Kafka: + topic: + CONNECTION-RECORD-COMPLETED-LOG: 3 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "{{ consumer_kafka_servers }}" + group.id: connection_record_completed_tsgv3 + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'tsg_galaxy_v3.connection_record_log_local' + username: 'default' + password: 'ceiec2019' + hosts: +{% for ip in groups.clickhouse %} +{% if (groups.clickhouse[0]) != ip and (groups.clickhouse[1]) != ip%} + - 'tcp://{{ ip }}:9001' +{% endif %} +{% endfor %} + bulk_actions: 100000 + flush_interval: 30 + concurrent: 2 + conn_max_life_time: 60 diff --git a/parcels/roles/services/galaxy-gohangout/templates/k2ck_gtpc_record_log_tsgv3.yml.j2 b/parcels/roles/services/galaxy-gohangout/templates/k2ck_gtpc_record_log_tsgv3.yml.j2 new file mode 100644 index 0000000..c5384e5 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/templates/k2ck_gtpc_record_log_tsgv3.yml.j2 @@ -0,0 +1,32 @@ +inputs: + - Kafka: + topic: + GTPC-RECORD-COMPLETED-LOG: 1 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "{{ consumer_kafka_servers }}" + group.id: gtpc_record_completed_tsgv3 + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'tsg_galaxy_v3.gtpc_record_log_local' + username: 'default' + password: 'ceiec2019' + conn_max_life_time: 60 + hosts: +{% for ip in groups.clickhouse %} +{% if (groups.clickhouse[0]) != ip and (groups.clickhouse[1]) != ip%} + - 'tcp://{{ ip }}:9001' +{% endif %} +{% endfor %} + bulk_actions: 100000 + flush_interval: 30 + concurrent: 2 diff --git a/parcels/roles/services/galaxy-gohangout/templates/k2ck_live_session_record_log_tsgv3.yml.j2 b/parcels/roles/services/galaxy-gohangout/templates/k2ck_live_session_record_log_tsgv3.yml.j2 new file mode 100644 index 0000000..70d107c --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/templates/k2ck_live_session_record_log_tsgv3.yml.j2 @@ -0,0 +1,32 @@ +inputs: + - Kafka: + topic: + LIVE-SESSION-RECORD-COMPLETED-LOG: 1 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "{{ consumer_kafka_servers }}" + group.id: live_session_completed_tsgv3 + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'tsg_galaxy_v3.live_session_record_log_local' + username: 'default' + password: 'ceiec2019' + conn_max_life_time: 60 + hosts: +{% for ip in groups.clickhouse %} +{% if (groups.clickhouse[0]) != ip and (groups.clickhouse[1]) != ip%} + - 'tcp://{{ ip }}:9001' +{% endif %} +{% endfor %} + bulk_actions: 100000 + flush_interval: 30 + concurrent: 2 diff --git a/parcels/roles/services/galaxy-gohangout/templates/k2ck_proxy_event_log_tsgv3.yml.j2 b/parcels/roles/services/galaxy-gohangout/templates/k2ck_proxy_event_log_tsgv3.yml.j2 new file mode 100644 index 0000000..d872007 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/templates/k2ck_proxy_event_log_tsgv3.yml.j2 @@ -0,0 +1,32 @@ +inputs: + - Kafka: + topic: + PROXY-EVENT-COMPLETED-LOG: 3 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "{{ consumer_kafka_servers }}" + group.id: proxy_event_completed_tsgv3 + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'tsg_galaxy_v3.proxy_event_log_local' + username: 'default' + password: 'ceiec2019' + hosts: +{% for ip in groups.clickhouse %} +{% if (groups.clickhouse[0]) != ip and (groups.clickhouse[1]) != ip%} + - 'tcp://{{ ip }}:9001' +{% endif %} +{% endfor %} + bulk_actions: 100000 + flush_interval: 30 + concurrent: 2 + conn_max_life_time: 60 diff --git a/parcels/roles/services/galaxy-gohangout/templates/k2ck_radius_onff_log_tsgv3.yml.j2 b/parcels/roles/services/galaxy-gohangout/templates/k2ck_radius_onff_log_tsgv3.yml.j2 new file mode 100644 index 0000000..7dd30dc --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/templates/k2ck_radius_onff_log_tsgv3.yml.j2 @@ -0,0 +1,32 @@ +inputs: + - Kafka: + topic: + RADIUS-ONFF-LOG: 3 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "{{ consumer_kafka_servers }}" + group.id: radius_onff_log_tsgv3 + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'tsg_galaxy_v3.radius_onff_log_local' + username: 'default' + password: 'ceiec2019' + hosts: +{% for ip in groups.clickhouse %} +{% if (groups.clickhouse[0]) != ip and (groups.clickhouse[1]) != ip%} + - 'tcp://{{ ip }}:9001' +{% endif %} +{% endfor %} + bulk_actions: 100000 + flush_interval: 30 + concurrent: 2 + conn_max_life_time: 60 diff --git a/parcels/roles/services/galaxy-gohangout/templates/k2ck_radius_record_log_tsgv3.yml.j2 b/parcels/roles/services/galaxy-gohangout/templates/k2ck_radius_record_log_tsgv3.yml.j2 new file mode 100644 index 0000000..e91ffa0 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/templates/k2ck_radius_record_log_tsgv3.yml.j2 @@ -0,0 +1,32 @@ +inputs: + - Kafka: + topic: + RADIUS-RECORD-COMPLETED-LOG: 3 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "{{ consumer_kafka_servers }}" + group.id: radius_record_completed_tsgv3 + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'tsg_galaxy_v3.radius_record_log_local' + username: 'default' + password: 'ceiec2019' + hosts: +{% for ip in groups.clickhouse %} +{% if (groups.clickhouse[0]) != ip and (groups.clickhouse[1]) != ip%} + - 'tcp://{{ ip }}:9001' +{% endif %} +{% endfor %} + bulk_actions: 100000 + flush_interval: 30 + concurrent: 2 + conn_max_life_time: 60 diff --git a/parcels/roles/services/galaxy-gohangout/templates/k2ck_security_event_log_tsgv3.yml.j2 b/parcels/roles/services/galaxy-gohangout/templates/k2ck_security_event_log_tsgv3.yml.j2 new file mode 100644 index 0000000..c640f32 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/templates/k2ck_security_event_log_tsgv3.yml.j2 @@ -0,0 +1,32 @@ +inputs: + - Kafka: + topic: + SECURITY-EVENT-COMPLETED-LOG: 3 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "{{ consumer_kafka_servers }}" + group.id: security_event_completed_tsgv3 + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'tsg_galaxy_v3.security_event_log_local' + username: 'default' + password: 'ceiec2019' + hosts: +{% for ip in groups.clickhouse %} +{% if (groups.clickhouse[0]) != ip and (groups.clickhouse[1]) != ip%} + - 'tcp://{{ ip }}:9001' +{% endif %} +{% endfor %} + bulk_actions: 100000 + flush_interval: 30 + concurrent: 2 + conn_max_life_time: 60 diff --git a/parcels/roles/services/galaxy-gohangout/templates/k2ck_sys_packet_capture_log_tsgv3.yml.j2 b/parcels/roles/services/galaxy-gohangout/templates/k2ck_sys_packet_capture_log_tsgv3.yml.j2 new file mode 100644 index 0000000..fb86005 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/templates/k2ck_sys_packet_capture_log_tsgv3.yml.j2 @@ -0,0 +1,32 @@ +inputs: + - Kafka: + topic: + SYS-PACKET-CAPTURE-COMPLETED-LOG: 3 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "{{ consumer_kafka_servers }}" + group.id: sys_packet_capture_completed_tsgv3 + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'tsg_galaxy_v3.sys_packet_capture_log_local' + username: 'default' + password: 'ceiec2019' + hosts: +{% for ip in groups.clickhouse %} +{% if (groups.clickhouse[0]) != ip and (groups.clickhouse[1]) != ip%} + - 'tcp://{{ ip }}:9001' +{% endif %} +{% endfor %} + bulk_actions: 100000 + flush_interval: 30 + concurrent: 2 + conn_max_life_time: 60 diff --git a/parcels/roles/services/galaxy-gohangout/templates/k2ck_transaction_record_log_tsgv3.yml.j2 b/parcels/roles/services/galaxy-gohangout/templates/k2ck_transaction_record_log_tsgv3.yml.j2 new file mode 100644 index 0000000..e6381dc --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/templates/k2ck_transaction_record_log_tsgv3.yml.j2 @@ -0,0 +1,32 @@ +inputs: + - Kafka: + topic: + TRANSACTION-RECORD-COMPLETED-LOG: 1 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "{{ consumer_kafka_servers }}" + group.id: transaction_record_completed_tsgv3 + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'tsg_galaxy_v3.transaction_record_log_local' + username: 'default' + password: 'ceiec2019' + conn_max_life_time: 60 + hosts: +{% for ip in groups.clickhouse %} +{% if (groups.clickhouse[0]) != ip and (groups.clickhouse[1]) != ip%} + - 'tcp://{{ ip }}:9001' +{% endif %} +{% endfor %} + bulk_actions: 100000 + flush_interval: 30 + concurrent: 2 diff --git a/parcels/roles/services/galaxy-gohangout/templates/k2ck_voip_record_log_tsgv3.yml.j2 b/parcels/roles/services/galaxy-gohangout/templates/k2ck_voip_record_log_tsgv3.yml.j2 new file mode 100644 index 0000000..2496197 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/templates/k2ck_voip_record_log_tsgv3.yml.j2 @@ -0,0 +1,32 @@ +inputs: + - Kafka: + topic: + VOIP-RECORD-COMPLETED-LOG: 1 + #assign: + # weblog: [0,9] + codec: json + consumer_settings: + bootstrap.servers: "{{ consumer_kafka_servers }}" + group.id: voip_record_completed_tsgv3 + max.partition.fetch.bytes: '10485760' + auto.commit.interval.ms: '5000' + # from.beginning: 'true' + # sasl.mechanism: PLAIN + # sasl.user: admin + # sasl.password: admin-secret + +outputs: + - Clickhouse: + table: 'tsg_galaxy_v3.voip_record_log_local' + username: 'default' + password: 'ceiec2019' + conn_max_life_time: 60 + hosts: +{% for ip in groups.clickhouse %} +{% if (groups.clickhouse[0]) != ip and (groups.clickhouse[1]) != ip%} + - 'tcp://{{ ip }}:9001' +{% endif %} +{% endfor %} + bulk_actions: 100000 + flush_interval: 30 + concurrent: 2 diff --git a/parcels/roles/services/galaxy-gohangout/vars/main.yml b/parcels/roles/services/galaxy-gohangout/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/services/galaxy-gohangout/vars/main.yml diff --git a/parcels/roles/services/galaxy-hos-keepalive/tasks/main.yml b/parcels/roles/services/galaxy-hos-keepalive/tasks/main.yml new file mode 100644 index 0000000..a7b8ddd --- /dev/null +++ b/parcels/roles/services/galaxy-hos-keepalive/tasks/main.yml @@ -0,0 +1,94 @@ +#创建文件夹 +- name: create keepalived package path:{{ keepalived_package_path }} + file: + state: directory + path: '{{ keepalived_package_path }}' + +#拷贝keepalive安装包 +- name: copy keepalived-1.4.0.tar.gz + copy: + src: '{{ package_path }}/keepalived-1.4.0.tar.gz' + dest: '{{ keepalived_package_path }}/' + force: true + backup: yes + +#拷贝keepalive基础环境包 +- name: copy keepalived-depend-rpm.zip + unarchive: + src: '{{ package_path }}/keepalived-depend-rpm.zip' + dest: '{{ keepalived_package_path }}/' +# copy: yes + register: copy_info + +#安装keepalive环境包 +- name: rpm install keepalived-depend-rpm + shell: rpm -ivh {{ keepalived_package_path }}/keepalived-depend-rpm/*.rpm --force --nodeps + when: copy_info.changed + +- name: master_ip to ansible variable + set_fact: master_ip="{{groups.hoskeepalive[0]}}" + +- name: backup_ip to ansible variable + set_fact: backup_ip="{{groups.hoskeepalive[1]}}" + +#拷贝keepalive配置文件 +- name: copy hos-keepalived.conf.master.j2 + template: + src: 'hos-keepalived.conf.master.j2' + dest: '{{ keepalived_package_path }}/keepalived.conf' + backup: yes + run_once: true + delegate_to: "{{ master_ip }}" + + #拷贝keepalive配置文件 +- name: copy hos-keepalived.conf.backup.j2 + template: + src: 'hos-keepalived.conf.backup.j2' + dest: '{{ keepalived_package_path }}/keepalived.conf' + backup: yes + run_once: true + delegate_to: "{{ backup_ip }}" + +- name: copy kp_daemon.sh + template: src=kp_daemon.sh.j2 dest={{ keepalived_package_path }}/kp_daemon.sh backup=yes mode=u+x + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ master_ip }}' } + - { ip: '{{ backup_ip }}' } + +- name: copy check_hos.sh + template: src=check_hos.sh.j2 dest={{ keepalived_package_path }}/check_hos.sh backup=yes mode=u+x + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ master_ip }}' } + - { ip: '{{ backup_ip }}' } + +- name: copy installKeepAlived.sh + template: src=installKeepAlived.sh.j2 dest={{ keepalived_package_path }}/installKeepAlived.sh backup=yes mode=u+x + register: copy_sh_info + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ master_ip }}' } + - { ip: '{{ backup_ip }}' } + +- name: exec installKeepAlived.sh + shell: cd {{ keepalived_package_path }} && ./installKeepAlived.sh + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ master_ip }}' } + - { ip: '{{ backup_ip }}' } + when: copy_sh_info.changed + +- name: Ansible delete {{ keepalived_package_path }} + file: + path: "{{ keepalived_package_path }}" + state: absent + diff --git a/parcels/roles/services/galaxy-hos-keepalive/templates/check_hos.sh.j2 b/parcels/roles/services/galaxy-hos-keepalive/templates/check_hos.sh.j2 new file mode 100644 index 0000000..65d92bb --- /dev/null +++ b/parcels/roles/services/galaxy-hos-keepalive/templates/check_hos.sh.j2 @@ -0,0 +1,26 @@ +#!/bin/bash + +#检测nginx服务是否存在脚本，若不存在则启动、启动失败则停止keepalive。 + +function checkNginx() { +NGINX_COUNT=$(ps -ef | grep nginx | grep -v grep | wc -l) +#判断是否都挂掉了 +if [ $NGINX_COUNT -eq 0 ] +then + #如果挂掉了，就启动 + cd {{ install_path }}/{{ keepalived_soft_home_path }}/galaxy-hos-nginx && docker-compose down + cd {{ install_path }}/{{ keepalived_soft_home_path }}/galaxy-hos-nginx && docker-compose up -d + #等5秒钟后，再次查看是否 启动成功 + sleep 5 + #如果没有启动起来，就直接干掉keepalived + NGINX_COUNT=$(ps -ef | grep nginx | grep -v grep | wc -l) + if [ $NGINX_COUNT -eq 0 ] + then + echo "干掉keepalived" + killall keepalived + fi +fi +} + +checkNginx + diff --git a/parcels/roles/services/galaxy-hos-keepalive/templates/hos-keepalived.conf.backup.j2 b/parcels/roles/services/galaxy-hos-keepalive/templates/hos-keepalived.conf.backup.j2 new file mode 100644 index 0000000..e669eeb --- /dev/null +++ b/parcels/roles/services/galaxy-hos-keepalive/templates/hos-keepalived.conf.backup.j2 @@ -0,0 +1,48 @@ +! Configuration File for keepalived + + +#其实配置文件中主要要修改的选项没有很多，以下几个参数要注意 +#interface 网卡名称 +#priority 150 #优先级，同一个实例下，MASTER高于BACKUP +####下面这两项已经修改好了,建议不要修改了 +#route_id XXX #MASTER和BACKUP不同 +#virtual_router_id 51 #同一个实例下，MASTER和BACKUP相同 + +#监控服务.NGINX mysql等 +vrrp_script chk_hos { + script "/etc/keepalived/check_hos.sh" + #每2s检查一次 + interval 2 + #每次检查-20 + weight -20 +} + +#VRRP实例定义块 +vrrp_instance VI_1 { +#状态只有MASTER和BACKUP两种，并且要大写，MASTER为工作状态，BACKUP是备用状态。 +state BACKUP +#网卡名称 +interface {{ hos_keepalive_interface }} +#虚拟路由标识，同一个vrrp_instance的MASTER和BACKUP的vitrual_router_id 是一致的。 +virtual_router_id {{ hos_keepalive_router_id }} +#优先级，同一个vrrp_instance的MASTER优先级必须比BACKUP高。 +priority 100 +#MASTER 与BACKUP 负载均衡器之间同步检查的时间间隔，单位为秒。 +advert_int 1 +authentication { +#验证authentication。包含验证类型和验证密码。类型主要有PASS、AH 两种，通常使用的类型为PASS +auth_type PASS +#据说AH 使用时有问题。验证密码为明文，同一vrrp 实例MASTER 与BACKUP 使用相同的密码才能正常通信 +auth_pass 1111 +} +#触发的脚本 +track_script { +#检测脚本，上面配置的 + chk_hos +} +virtual_ipaddress { +#虚拟ip地址,可以有多个地址，每个地址占一行，不需要子网掩码 + {{ hos_keepalive_host }} +} +} + diff --git a/parcels/roles/services/galaxy-hos-keepalive/templates/hos-keepalived.conf.master.j2 b/parcels/roles/services/galaxy-hos-keepalive/templates/hos-keepalived.conf.master.j2 new file mode 100644 index 0000000..78aafcb --- /dev/null +++ b/parcels/roles/services/galaxy-hos-keepalive/templates/hos-keepalived.conf.master.j2 @@ -0,0 +1,48 @@ +! Configuration File for keepalived + + +#其实配置文件中主要要修改的选项没有很多，以下几个参数要注意 +#interface 网卡名称 +#priority 150 #优先级，同一个实例下，MASTER高于BACKUP +####下面这两项已经修改好了,建议不要修改了 +#route_id XXX #MASTER和BACKUP不同 +#virtual_router_id 51 #同一个实例下，MASTER和BACKUP相同 + +#监控服务.NGINX mysql等 +vrrp_script chk_hos { + script "/etc/keepalived/check_hos.sh" + #每2s检查一次 + interval 2 + #每次检查-20 + weight -20 +} + +#VRRP实例定义块 +vrrp_instance VI_1 { +#状态只有MASTER和BACKUP两种，并且要大写，MASTER为工作状态，BACKUP是备用状态。 +state MASTER +#网卡名称 +interface {{ hos_keepalive_interface }} +#虚拟路由标识，同一个vrrp_instance的MASTER和BACKUP的vitrual_router_id 是一致的。 +virtual_router_id {{ hos_keepalive_router_id }} +#优先级，同一个vrrp_instance的MASTER优先级必须比BACKUP高。 +priority 150 +#MASTER 与BACKUP 负载均衡器之间同步检查的时间间隔，单位为秒。 +advert_int 1 +authentication { +#验证authentication。包含验证类型和验证密码。类型主要有PASS、AH 两种，通常使用的类型为PASS +auth_type PASS +#据说AH 使用时有问题。验证密码为明文，同一vrrp 实例MASTER 与BACKUP 使用相同的密码才能正常通信 +auth_pass 1111 +} +#触发的脚本 +track_script { +#检测脚本，上面配置的 + chk_hos +} +virtual_ipaddress { +#虚拟ip地址,可以有多个地址，每个地址占一行，不需要子网掩码 + {{ hos_keepalive_host }} +} +} + diff --git a/parcels/roles/services/galaxy-hos-keepalive/templates/installKeepAlived.sh.j2 b/parcels/roles/services/galaxy-hos-keepalive/templates/installKeepAlived.sh.j2 new file mode 100644 index 0000000..f6c1947 --- /dev/null +++ b/parcels/roles/services/galaxy-hos-keepalive/templates/installKeepAlived.sh.j2 @@ -0,0 +1,102 @@ +#!/bin/bash + + +#安装master还是backup +installMasterOrBackup=master +#keepalived安装包的名称 +keepalivedTarName=keepalived-1.4.0.tar.gz + +#wget -P /home/soft/keepalived/ http://www.keepalived.org/software/keepalived-1.4.0.tar.gz +#下载keepalived.tar到/home/soft/keepalived/目录下 +#守护进程名称 +keepalivedDaeonName=kp_daemon.sh +#master的keepalived.conf文件名称 +keepalivedMasterConfName=keepalived.conf +#bakcup的keepalived.conf文件名称 +#keepalivedBackupConfName=keepalived.conf +#检测nginx是否不用,不可用将nginx启动,在不可用杀死keepalived +checkShell=check_hos.sh + +#################################################################下面的写法,建议不要改动了################################################################# +#keepalived安装包的绝对路径 +keepalivedTarPath=`pwd`/$keepalivedTarName +#守护进程所在的目录,根据需要自行改动(会自动复制到keepalivedTarPath/sbin目录下,并添加到开机自启动中) +keepalivedDaeonPath=`pwd`/$keepalivedDaeonName +#将keepalived.tar解压到哪个目录下 +keepalivedBagPath=`pwd`/keepalived +checkShellPath=`pwd`/$checkShell +#判断是安装master还是backup +#if [ "$installMasterOrBackup" == "master" ];then + #keepalived.conf的文件路径 + keepalivedConfPath=`pwd`/$keepalivedMasterConfName +#else + #keepalived.conf的文件路径 +# keepalivedConfPath=`pwd`/$keepalivedBackupConfName +#fi + +#keepalived安装目录 +keepalivedInstallPath=/etc/keepalived +#keepalivedDaeonName=${keepalivedDaeonPath##*/} +#yum -y install gcc gcc-c++ +#yum -y install libnfnetlink-devel zlib zlib-devel openssl openssl-devel pcre pcre-devel + +#rm -rf $keepalivedBagPath + +installKP(){ + echo "开始解压 $keepalivedTarPath 到 $keepalivedBagPath" + ##--strip-components 1 + tar -xzf $keepalivedTarPath -C $keepalivedBagPath --strip-components 1 + echo "解压 $keepalivedTarPath 到 $keepalivedBagPath成功" + #cd $keepalivedBagPath && ./configure --prefix=$keepalivedInstallPath && make && make install + cd $keepalivedBagPath && ./configure && make && make install + #创建keepalived配置文件目录 + mkdir /etc/keepalived + #拷贝配置文件到/etc/keepalived目录下 + #`\cp -f $keepalivedInstallPath/etc/keepalived/keepalived.conf /etc/keepalived/ + `\cp -f $keepalivedConfPath /etc/keepalived/keepalived.conf` + #复制keepalived脚本到/etc/init.d/ 目录(开机自启动) + `\cp -f $keepalivedBagPath/keepalived/etc/init.d/keepalived /etc/init.d/keepalived` + + + + #复制keepalived守护脚本到$keepalivedInstallPath目录下 + echo "复制keepalived守护脚本到$keepalivedInstallPath目录下" + `\cp -f $keepalivedDaeonPath $keepalivedInstallPath/$keepalivedDaeonName` + #复制keepalived监控脚本到$keepalivedInstallPath目录下 + `\cp -f $checkShellPath $keepalivedInstallPath/$checkShell` + + chmod +x $keepalivedInstallPath/$keepalivedDaeonName + echo "将$keepalivedInstallPath/$keepalivedDaeonName守护进程添加到开机自启动中" + chmod +x /etc/rc.d/rc.local + echo -e "\n#设置$keepalivedDaeonName守护脚本开机自启动" >> /etc/rc.d/rc.local + echo "nohup $keepalivedInstallPath/$keepalivedDaeonName >/dev/null 2>&1 &" >> /etc/rc.d/rc.local + echo "将$keepalivedInstallPath/$keepalivedDaeonName守护进程添加到开机自启动成功" + + echo "开始启动$keepalivedInstallPath/$keepalivedDaeonName守护进程" + nohup $keepalivedInstallPath/$keepalivedDaeonName >/dev/null 2>&1 & + + rm -rf $keepalivedBagPath +} + + + + +if [ ! -d $keepalivedBagPath ]; then + echo "$keepalivedBagPath目录不存在,开始创建$keepalivedBagPath目录" + mkdir -p $keepalivedBagPath + installKP +else + count=`ls $keepalivedBagPath| wc -w` + if [ $count -ne 0 ] + then + echo "安装nginx失败!!!$keepalivedBagPath有其他文件,请将文件备份到其他目录或修改nginx安装目录" + else + installKP + fi +fi + + + + + + diff --git a/parcels/roles/services/galaxy-hos-keepalive/templates/kp_daemon.sh.j2 b/parcels/roles/services/galaxy-hos-keepalive/templates/kp_daemon.sh.j2 new file mode 100644 index 0000000..2d46f1e --- /dev/null +++ b/parcels/roles/services/galaxy-hos-keepalive/templates/kp_daemon.sh.j2 @@ -0,0 +1,23 @@ +#!/bin/bash + +keepalivedPath=/etc/keepalived +pidFile=/var/run/keepalived.pid +logPath=$keepalivedPath/keepalived_daemon.log + +while true;do + NGINX_COUNT=$(ps -ef | grep nginx | grep -v grep | wc -l) + + if [[ $NGINX_COUNT -eq "0" ]] + then + echo "时间：`date '+%Y-%m-%d %T'`galaxy-hos-nginx服务进程不存在,关闭keepalived进程" >> $logPath + systemctl stop keepalived + elif [ ! -f $pidFile ]; then + echo "时间：`date '+%Y-%m-%d %T'`keepalived进程不存在,如果服务都健康,开始启动keepalived" >> $logPath + if [[ $NGINX_COUNT -gt "0" ]] #hos都不存在停止keepalived + then + systemctl start keepalived + fi + fi + sleep 5 +done + diff --git a/parcels/roles/services/galaxy-hos-keepalive/vars/main.yml b/parcels/roles/services/galaxy-hos-keepalive/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/services/galaxy-hos-keepalive/vars/main.yml diff --git a/parcels/roles/services/galaxy-hos-nginx/tasks/main.yml b/parcels/roles/services/galaxy-hos-nginx/tasks/main.yml new file mode 100644 index 0000000..26fe1d8 --- /dev/null +++ b/parcels/roles/services/galaxy-hos-nginx/tasks/main.yml @@ -0,0 +1,115 @@ +- name: stop and remove {{ hos_nginx_image_container_name }} container + docker_container: + name: '{{ hos_nginx_image_container_name }}' + state: absent + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.hosnginx[0] }}' } + - { ip: '{{ groups.hosnginx[1] }}' } + +- name: remove old {{ hos_nginx_hos_service_image_name }} image + docker_image: + name: '{{ hos_nginx_hos_service_image_name }}' + tag: '{{ hos_nginx_hos_service_image_tag_name }}' + state: absent + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.hosnginx[0] }}' } + - { ip: '{{ groups.hosnginx[1] }}' } + +- name: create {{ hos_nginx_volume_path }} + file: + state: directory + path: '{{ install_path }}/{{ hos_nginx_volume_path }}' + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.hosnginx[0] }}' } + - { ip: '{{ groups.hosnginx[1] }}' } + +- name: create {{ install_path }}/{{ hos_nginx_soft_home_path }} + file: + state: directory + path: '{{ install_path }}/{{ hos_nginx_soft_home_path }}' + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.hosnginx[0] }}' } + - { ip: '{{ groups.hosnginx[1] }}' } + +- name: copy {{ hos_nginx_hos_service_image_tar_name }} + copy: + src: '{{ package_path }}/{{ hos_nginx_hos_service_image_tar_name }}' + dest: '{{ install_path }}/{{ hos_nginx_soft_home_path }}/' + force: true + backup: yes + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.hosnginx[0] }}' } + - { ip: '{{ groups.hosnginx[1] }}' } + +- name: load new {{ hos_nginx_hos_service_image_name }} image from {{ hos_nginx_hos_service_image_tar_name }} + docker_image: + name: '{{ hos_nginx_hos_service_image_name }}' + tag: '{{ hos_nginx_hos_service_image_tag_name }}' + load_path: '{{ install_path }}/{{ hos_nginx_soft_home_path }}/{{ hos_nginx_hos_service_image_tar_name }}' + source: load + force_tag: yes + force_source: yes + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.hosnginx[0] }}' } + - { ip: '{{ groups.hosnginx[1] }}' } + +- name: change the image tag + shell: docker tag nginx-metrics:{{ hos_nginx_hos_service_image_tag_name }} {{ hos_nginx_hos_service_image_name }}:{{ hos_nginx_hos_service_image_tag_name }} + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.hosnginx[0] }}' } + - { ip: '{{ groups.hosnginx[1] }}' } + +- name: copy {{ hos_nginx_hos_service_image_name }} docker-compose.yml + template: + src: docker-compose.yml.j2 + dest: '{{ install_path }}/{{ hos_nginx_soft_home_path }}/docker-compose.yml' + backup: yes + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.hosnginx[0] }}' } + - { ip: '{{ groups.hosnginx[1] }}' } + +- name: copy {{ hos_nginx_hos_service_image_name }} nginx.conf + template: + src: nginx.conf.j2 + dest: '{{ install_path }}/{{ hos_nginx_volume_path }}/nginx.conf' + backup: yes + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.hosnginx[0] }}' } + - { ip: '{{ groups.hosnginx[1] }}' } + +- name: start {{ hos_nginx_image_container_name }} container + docker_compose: + project_src: '{{ install_path }}/{{ hos_nginx_soft_home_path }}' + run_once: true + delegate_facts: true + delegate_to: "{{ item.ip }}" + with_items: + - { ip: '{{ groups.hosnginx[0] }}' } + - { ip: '{{ groups.hosnginx[1] }}' } diff --git a/parcels/roles/services/galaxy-hos-nginx/templates/docker-compose.yml.j2 b/parcels/roles/services/galaxy-hos-nginx/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..f994151 --- /dev/null +++ b/parcels/roles/services/galaxy-hos-nginx/templates/docker-compose.yml.j2 @@ -0,0 +1,14 @@ +version: '3' +services: + nginx: + image: {{ hos_nginx_hos_service_image_name }}:{{ hos_nginx_hos_service_image_tag_name }} + container_name: {{ hos_nginx_image_container_name }} + restart: always + ports: + - 80:80 + volumes: + - {{ install_path }}/{{ hos_nginx_volume_path }}/logs:/usr/local/nginx/logs + - {{ install_path }}/{{ hos_nginx_volume_path }}/nginx.conf:/usr/local/nginx/conf/nginx.conf + working_dir: /etc/nginx + command: /etc/nginx/nginx -g 'daemon off;' + network_mode: "host" diff --git a/parcels/roles/services/galaxy-hos-nginx/templates/nginx.conf.j2 b/parcels/roles/services/galaxy-hos-nginx/templates/nginx.conf.j2 new file mode 100644 index 0000000..1c0aab3 --- /dev/null +++ b/parcels/roles/services/galaxy-hos-nginx/templates/nginx.conf.j2 @@ -0,0 +1,37 @@ +worker_processes 4; + +events { + worker_connections 1024; +} + + +stream { + + # preread_buffer_size 64k; + # preread_timeout 90s; + # proxy_protocol_timeout 300s; + # resolver_timeout 10s; + + log_format proxy '$remote_addr [$time_local] ' + '$protocol $status $bytes_sent $bytes_received ' + '$session_time "$upstream_addr" ' + '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"'; + + access_log /usr/local/nginx/logs/access.log proxy ; + open_log_file_cache off; + include /etc/nginx/conf.d/*.stream; + + upstream hos { +{% for dev_info in groups.hos %} + server {{dev_info}}:8186 max_fails=1 fail_timeout=60s; +{% endfor %} + } + + server { + listen 9098; + proxy_connect_timeout 20s; + proxy_timeout 20s; + proxy_pass hos; + } +} + diff --git a/parcels/roles/services/galaxy-hos-nginx/vars/main.yml b/parcels/roles/services/galaxy-hos-nginx/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/services/galaxy-hos-nginx/vars/main.yml diff --git a/parcels/roles/services/galaxy-hos-service/tasks/load_hos.yml b/parcels/roles/services/galaxy-hos-service/tasks/load_hos.yml new file mode 100644 index 0000000..b067022 --- /dev/null +++ b/parcels/roles/services/galaxy-hos-service/tasks/load_hos.yml @@ -0,0 +1,79 @@ +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181" + when: '(groups.zookeeper|length) == 3' + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181,{{groups.zookeeper[3]}}:2181,{{groups.zookeeper[4]}}:2181" + when: '(groups.zookeeper|length) >= 5' + +- name: stop and remove {{ hos_service_image_container_name }} container + docker_container: + name: '{{ hos_service_image_container_name }}' + state: absent + +- name: create hos config + file: + state: directory + path: '{{ data_path }}/{{ hos_volume_path }}/config' + +- name: create hos workerspace + file: + state: directory + path: '{{ data_path }}/{{ hos_service_soft_home_path }}' + +- name: copy {{ image_tar_name }} + copy: + src: '{{ package_path }}/{{ image_tar_name }}' + dest: '{{ data_path }}/{{ hos_service_soft_home_path }}/{{ image_tar_name }}' + force: true + backup: yes + +- name: load new {{ image_name }} image from {{ image_tar_name }} + docker_image: + name: '{{ image_name }}' + tag: '{{ image_tag_name }}' + load_path: '{{ data_path }}/{{ hos_service_soft_home_path }}/{{ image_tar_name }}' + source: load + force_tag: yes + force_source: yes + async: 300 + +- name: change the image tag + shell: docker tag galaxy-hos-service:{{ image_tag_name }} {{ image_name }}:{{ image_tag_name }} + + +- name: copy {{ image_name }} docker-compose.yml + template: + src: docker-compose.yml.j2 + dest: '{{ data_path }}/{{ hos_service_soft_home_path }}/docker-compose.yml' + backup: yes + +- name: copy {{ image_name }} application.yml.j2 + template: + src: application.yml.j2 + dest: '{{ data_path }}/{{ hos_volume_path }}/config/application.yml' + backup: yes + +- name: copy {{ image_name }} log4j2-dev.xml.j2 + template: + src: log4j2-dev.xml.j2 + dest: '{{ data_path }}/{{ hos_volume_path }}/config/log4j2-dev.xml' + backup: yes + +- name: copy {{ image_name }} galaxy-hos-service.j2 + template: + src: galaxy-hos-service.j2 + dest: '{{ data_path }}/{{ hos_service_soft_home_path }}/galaxy-hos-service' + backup: yes + +- name: push config + shell: 'curl --data-urlencode content="`cat {{ data_path }}/{{ hos_service_soft_home_path }}/galaxy-hos-service`" --request POST "http://{{ gateway_keepalive_host }}:8848/nacos/v1/cs/configs?username=nacos&password={{ nacos_pin }}&tenant={{ services_config_namespace }}&group=Galaxy&dataId=galaxy-hos-service.yml&appName=galaxy-hos-service&type=yaml"' + register: change_out + +- fail: + msg: "galaxy-hos-service 配置提交失败" + when: change_out.stdout != 'true' + +- name: start {{ hos_service_image_container_name }} container + docker_compose: + project_src: '{{ data_path }}/{{ hos_service_soft_home_path }}' diff --git a/parcels/roles/services/galaxy-hos-service/tasks/main.yml b/parcels/roles/services/galaxy-hos-service/tasks/main.yml new file mode 100644 index 0000000..ab2e4a5 --- /dev/null +++ b/parcels/roles/services/galaxy-hos-service/tasks/main.yml @@ -0,0 +1,2 @@ +#- include: update_python3_and_docker_py.yml +- include: load_hos.yml diff --git a/parcels/roles/services/galaxy-hos-service/templates/application.yml.j2 b/parcels/roles/services/galaxy-hos-service/templates/application.yml.j2 new file mode 100644 index 0000000..e9445d8 --- /dev/null +++ b/parcels/roles/services/galaxy-hos-service/templates/application.yml.j2 @@ -0,0 +1,20 @@ +nacos: + config: + type: yaml + server-addr: {{ gateway_keepalive_host }}:8848 + namespace: {{ services_config_namespace }} + data-id: galaxy-hos-service.yml + auto-refresh: true + group: Galaxy + username: nacos + password: {{ nacos_pin }} + bootstrap: + enable: true + log: + enable: true +spring: + profiles: + active: dev +logging: + config: ./config/log4j2-dev.xml + diff --git a/parcels/roles/services/galaxy-hos-service/templates/docker-compose.yml.j2 b/parcels/roles/services/galaxy-hos-service/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..729bd71 --- /dev/null +++ b/parcels/roles/services/galaxy-hos-service/templates/docker-compose.yml.j2 @@ -0,0 +1,17 @@ +version: '2' + +services: + galaxy-hos-service: + image: {{ image_name }}:{{ image_tag_name }} + container_name: {{ hos_service_image_container_name }} + environment: + JAVA_OPTS: "{{ hos_java_opts }} -Xss256k -XX:MetaspaceSize=128m -XX:MaxPermSize=128m -XX:SurvivorRatio=2 -XX:+UseParNewGC -XX:+CMSParallelRemarkEnabled -XX:MaxTenuringThreshold=15 -XX:+UseCMSCompactAtFullCollection -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70" + ports: + - "9098:9098" + volumes: + - "{{ data_path }}/{{ hos_volume_path }}/config:/home/tsg/galaxy/galaxy-hos-service/config" + - "{{ data_path }}/{{ hos_volume_path }}/logs:/home/tsg/galaxy/galaxy-hos-service/logs" + - "/etc/localtime:/etc/localtime:ro" + - "/etc/timezone:/etc/timezone:ro" + restart: always + network_mode: "host" diff --git a/parcels/roles/services/galaxy-hos-service/templates/galaxy-hos-service.j2 b/parcels/roles/services/galaxy-hos-service/templates/galaxy-hos-service.j2 new file mode 100644 index 0000000..ec03155 --- /dev/null +++ b/parcels/roles/services/galaxy-hos-service/templates/galaxy-hos-service.j2 @@ -0,0 +1,76 @@ +#服务端口 +server: + port: 8186 +#tomcat缓存大小，单位KB系统默认10M，配置10g +tomcat: + cacheMaxSize: {{ hos_tomcat_cache_size }} +#hbase参数 +hbase: + zookeeper_quorum: {{ zookeeper_servers }} + zookeeper_property_clientPort: 2181 + zookeeper_znode_parent: /hbase + client_retries_number: 1 + rpc_timeout: 100000 + connect_pool: 300 + client_write_buffer: 10485760 + client_keyvalue_maxsize: 1024000000 + #批量获取数量 + get_batch: 200 + #hbase索引表前缀，前缀为以下的都为索引表 + time_index_table_prefix: index_time_ + filename_index_table_prefix: index_filename_ + partfile_index_table_prefix: index_partfile_ + system_bucket_meta: system:bucket_meta + #创建表预分区时的分区，为空则不分区 + region_start_key: 1,2,3,4,5,6,7,8,9,a,b,c,d,e,f + filename_head: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f + #获取文件大小的目录 + hbasePath: /home/tsg/galaxy/data + #1是集群0是单机，主要针对存储配额获取方式 + standone: 1 + #hadoop集群namenode节点 + namenodes: {{groups.hadoop[0]}},{{groups.hadoop[1]}} + #hadoop端口 + hadoop_port: 9000 + #用户白名单（hbase的namespace），获取存储配额 +#是否打开验证，0打开，打开需要使用S3身份验证或者token访问服务 +auth: + open: 0 +hos: + #批量删除对象的最大数量 + deleteMultipleNumber: 1000 + #获取对象列表等操作的最大值 + maxResultLimit: 10000 + #分块上传的最大分块数 + maxPartNumber: 1000 + #追加上传的最大次数 + maxPosition: 100000 + #存放文件元数据的请求头 + metaHeader: x-hos-meta-message + #是否打开对象列表查询功能，1打开 + simple: 1 + #hos验证 + token: c21f969b5f03d33d43e04f8f136e7682 + users: default + #元数据存储占比 + metaProportion: 0.03 +#设置上传文件大小的最大值 +spring: + servlet: + multipart: + max-file-size: 1024MB + max-request-size: 1024MB +#Prometheus参数 + application: + name: HosServiceApplication +#Prometheus参数 +management: + endpoints: + web: + exposure: + include: '*' + metrics: + tags: + application: ${spring.application.name} +logging: + config: ./config/log4j2-dev.xml diff --git a/parcels/roles/services/galaxy-hos-service/templates/log4j2-dev.xml.j2 b/parcels/roles/services/galaxy-hos-service/templates/log4j2-dev.xml.j2 new file mode 100644 index 0000000..7f02b75 --- /dev/null +++ b/parcels/roles/services/galaxy-hos-service/templates/log4j2-dev.xml.j2 @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="UTF-8"?> +<configuration> + + <!--日志打印相关参数配置--> + <Properties> + <!--每5M压缩日志文件--> + <property name="LOG_SIZE">5M</property> + <!--最多产生10个压缩文件--> + <property name="LOG_NUMS">10</property> + <!--日志打印等级--> + <property name="LOG_LEVEL">info</property> + <!--日志文件路径--> + <property name="LOG_PATH">logs</property> + <!--日志文件名称--> + <property name="LOG_FILE_NAME">galaxy-hos-service</property> + <!--日志打印格式--> + <property name="LOG_PATTERN">[%d{yyyy-MM-dd HH:mm:ss}] [%p] [Thread:%t] %l %x - %m%n</property> + </Properties> + + <appenders> + <Console name="consoleSystemOutAppender" target="SYSTEM_OUT"> + <ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/> + <PatternLayout pattern="${LOG_PATTERN}"/> + </Console> + + <RollingFile name="rollingFileAllAppender" + fileName="${LOG_PATH}/${LOG_FILE_NAME}.log" + filePattern="${LOG_PATH}/history/$${date:yyyy-MM-dd}/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz"> + <PatternLayout pattern="${LOG_PATTERN}"/> + <Policies> + <SizeBasedTriggeringPolicy size="${LOG_SIZE}"/> + <TimeBasedTriggeringPolicy interval="1" modulate="true"/> + </Policies> + <Filters> + <ThresholdFilter level="all" onMatch="ACCEPT" onMismatch="DENY"/> + </Filters> + <DefaultRolloverStrategy max="${LOG_NUMS}"> + <Delete basePath="${LOG_PATH}/history" maxDepth="1"> + <IfFileName glob="*.log.gz"> + <IfLastModified age="90d"> + <IfAny> + <IfAccumulatedFileSize exceeds="200 GB" /> + </IfAny> + </IfLastModified> + </IfFileName> + </Delete> + </DefaultRolloverStrategy> + </RollingFile> + </appenders> + <loggers> + <root level="${LOG_LEVEL}"> + <appender-ref ref="consoleSystemOutAppender"/> + <appender-ref ref="rollingFileAllAppender"/> + </root> + </loggers> +</configuration> diff --git a/parcels/roles/services/galaxy-hos-service/templates/tmp/application.properties.j2 b/parcels/roles/services/galaxy-hos-service/templates/tmp/application.properties.j2 new file mode 100644 index 0000000..ef6f3b2 --- /dev/null +++ b/parcels/roles/services/galaxy-hos-service/templates/tmp/application.properties.j2 @@ -0,0 +1,57 @@ +#hbase参数 +hbase.zookeeper_quorum={{ zookeeper_servers }} +hbase.zookeeper_property_clientPort=2181 +hbase.zookeeper_znode_parent=/hbase +hbase.client_retries_number=1 +hbase.rpc_timeout=100000 +hbase.connect_pool=300 +hbase.client_write_buffer=10485760 +hbase.client_keyvalue_maxsize=1024000000 +#hbase索引表前缀，前缀为以下的都为索引表 +hbase.time_index_table_prefix=index_time_ +hbase.filename_index_table_prefix=index_filename_ +hbase.partfile_index_table_prefix=index_partfile_ +hbase.system_bucket_meta=system:bucket_meta +#创建表预分区时的分区，为空则不分区 +hbase.region_start_key=1,2,3,4,5,6,7,8,9,a,b,c,d,e,f +hbase.filename_head= 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f + +#设置上传文件大小的最大值 +spring.servlet.multipart.max-file-size=1024MB +spring.servlet.multipart.max-request-size=1024MB + +#批量删除对象的最大数量 +hos.deleteMultipleNumber=1000 +#获取对象列表等操作的最大值 +hos.maxResultLimit=10000 +#分块上传的最大分块数 +hos.maxPartNumber=1000 +#追加上传的最大次数 +hos.maxPosition=10000 +#存放文件元数据的请求头 +hos.metaHeader=x-hos-meta-message +#是否打开对象列表查询功能，1打开 +hos.simple=1 +#是否打开验证，0打开，打开需要使用S3身份验证或者token访问服务 +auth.open=0 +#hos验证 +hos.token=c21f969b5f03d33d43e04f8f136e7682 +#tomcat缓存大小，单位KB，配置10g +tomcat.cacheMaxSize={{ hos_tomcat_cache_size }} +#服务端口 +server.port=8185 +#Prometheus参数 +spring.application.name=HosServiceApplication +management.endpoints.web.exposure.include=* +management.metrics.tags.application=${spring.application.name} +logging.config= ./config/log4j2-dev.xml +#获取文件大小的目录 +hbase.hbasePath=/hbase +#1是集群0是单机，主要针对存储配额获取方式 +hbase.standone=1 +#hadoop集群namenode节点 +hbase.namenodes={{ groups.hadoop[0] }},{{ groups.hadoop[1] }} +#hadoop端口 +hbase.hadoop_port=9000 +#用户白名单（hbase的namespace），获取存储配额 +hos.users=default diff --git a/parcels/roles/services/galaxy-hos-service/templates/tmp/galaxy-hos-service.j2 b/parcels/roles/services/galaxy-hos-service/templates/tmp/galaxy-hos-service.j2 new file mode 100644 index 0000000..77f532a --- /dev/null +++ b/parcels/roles/services/galaxy-hos-service/templates/tmp/galaxy-hos-service.j2 @@ -0,0 +1,74 @@ +#服务端口 +server: + port: 8186 +#tomcat缓存大小，单位KB系统默认10M，配置10g +tomcat: + cacheMaxSize: {{ hos_tomcat_cache_size }} +#hbase参数 +hbase: + zookeeper_quorum: {{ zookeeper_servers }} + zookeeper_property_clientPort: 2181 + zookeeper_znode_parent: /hbase + client_retries_number: 1 + rpc_timeout: 100000 + connect_pool: 300 + client_write_buffer: 10485760 + client_keyvalue_maxsize: 1024000000 + #hbase索引表前缀，前缀为以下的都为索引表 + time_index_table_prefix: index_time_ + filename_index_table_prefix: index_filename_ + partfile_index_table_prefix: index_partfile_ + system_bucket_meta: system:bucket_meta + #创建表预分区时的分区，为空则不分区 + region_start_key: 1,2,3,4,5,6,7,8,9,a,b,c,d,e,f + filename_head: 0,1,2,3,4,5,6,7,8,9,a,b,c,d,e,f + #获取文件大小的目录 + hbasePath: /home/tsg/galaxy/data + #1是集群0是单机，主要针对存储配额获取方式 + standone: 1 + #hadoop集群namenode节点 + namenodes: {{groups.hadoop[0]}},{{groups.hadoop[1]}} + #hadoop端口 + hadoop_port: 9000 + #用户白名单（hbase的namespace），获取存储配额 +#是否打开验证，0打开，打开需要使用S3身份验证或者token访问服务 +auth: + open: 0 +hos: + #批量删除对象的最大数量 + deleteMultipleNumber: 1000 + #获取对象列表等操作的最大值 + maxResultLimit: 10000 + #分块上传的最大分块数 + maxPartNumber: 1000 + #追加上传的最大次数 + maxPosition: 10000 + #存放文件元数据的请求头 + metaHeader: x-hos-meta-message + #是否打开对象列表查询功能，1打开 + simple: 1 + #hos验证 + token: c21f969b5f03d33d43e04f8f136e7682 + users: default + #元数据存储占比 + metaProportion: 0.03 +#设置上传文件大小的最大值 +spring: + servlet: + multipart: + max-file-size: 1024MB + max-request-size: 1024MB +#Prometheus参数 + application: + name: HosServiceApplication +#Prometheus参数 +management: + endpoints: + web: + exposure: + include: '*' + metrics: + tags: + application: ${spring.application.name} +logging: + config: ./config/log4j2-dev.xml diff --git a/parcels/roles/services/galaxy-hos-service/vars/main.yml b/parcels/roles/services/galaxy-hos-service/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/services/galaxy-hos-service/vars/main.yml diff --git a/parcels/roles/services/galaxy-job-service/files/mysql b/parcels/roles/services/galaxy-job-service/files/mysql Binary files differnew file mode 100755 index 0000000..66af1a1 --- /dev/null +++ b/parcels/roles/services/galaxy-job-service/files/mysql diff --git a/parcels/roles/services/galaxy-job-service/tasks/load_admin.yml b/parcels/roles/services/galaxy-job-service/tasks/load_admin.yml new file mode 100644 index 0000000..d318423 --- /dev/null +++ b/parcels/roles/services/galaxy-job-service/tasks/load_admin.yml @@ -0,0 +1,111 @@ +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181" + when: '(groups.zookeeper|length) == 3' + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181,{{groups.zookeeper[3]}}:2181,{{groups.zookeeper[4]}}:2181" + when: '(groups.zookeeper|length) >= 5' + +- name: get job_admin_servers to ansible variable + set_fact: job_admin_servers="http://{{ groups.services[0] }}:8184/xxl-job-admin,http://{{ groups.services[1] }}:8184/xxl-job-admin" + when: '(groups.services|length) == 2' + +- name: get job_admin_servers to ansible variable + set_fact: job_admin_servers="http://{{ groups.services[0] }}:8184/xxl-job-admin,http://{{ groups.services[1] }}:8184/xxl-job-admin,http://{{ groups.services[2] }}:8184/xxl-job-admin" + when: '(groups.services|length) == 3' + +- name: get job_executor_servers to ansible variable + set_fact: job_executor_servers="http://{{ groups.services[0] }}:8886,http://{{ groups.services[1] }}:8886" + when: '(groups.services|length) == 2' + +- name: get job_executor_servers to ansible variable + set_fact: job_executor_servers="http://{{ groups.services[0] }}:8886,http://{{ groups.services[1] }}:8886,http://{{ groups.services[2] }}:8886" + when: '(groups.services|length) == 3' + + +- name: stop and remove {{ admin_image_container_name }} container + docker_container: + name: '{{ admin_image_container_name }}' + state: absent + +- name: remove old {{ admin_image_name }} image + docker_image: + name: '{{ admin_image_name }}' + tag: '{{ admin_image_tag_name }}' + state: absent + +- name: create {{ admin_volume_path }} + file: + state: directory + path: '{{ install_path }}/{{ admin_volume_path }}/config' + +- name: create {{ admin_soft_home_path }} + file: + state: directory + path: '{{ install_path }}/{{ admin_soft_home_path }}' + +- name: copy {{ admin_image_tar_name }} + copy: + src: '{{ {{ package_path }} }}/{{ admin_image_tar_name }}' + dest: '{{ install_path }}/{{ admin_soft_home_path }}/{{ admin_image_tar_name }}' + force: true + backup: yes + +- name: load new {{ admin_image_name }} image from {{ admin_image_tar_name }} + docker_image: + name: '{{ admin_image_name }}' + tag: '{{ admin_image_tag_name }}' + load_path: '{{ install_path }}/{{ admin_soft_home_path }}/{{ admin_image_tar_name }}' + source: load + force_tag: yes + force_source: yes + +- name: change the image tag + shell: docker tag galaxy-job-admin:{{ admin_image_tag_name }} {{ admin_image_name }}:{{ admin_image_tag_name }} + +- name: copy {{ admin_image_name }} admin_application.properties.j2 + template: + src: admin_application.properties.j2 + dest: '{{ install_path }}/{{ admin_volume_path }}/config/application.properties' + backup: yes + +- name: copy {{ admin_image_name }} logback.xml + template: + src: admin_logback.xml.j2 + dest: '{{ install_path }}/{{ admin_volume_path }}/config/logback.xml' + backup: yes + +- name: copy mysql + copy: + src: '../files/mysql' + dest: '/usr/bin/' + force: true + run_once: true + delegate_to: '{{groups.services[0]}}' + +- name: change mode mysql + shell: chmod +x /usr/bin/mysql + run_once: true + delegate_to: '{{groups.services[0]}}' + +- name: copy {{ admin_image_name }} xxl_job.sql.j2 + template: + src: xxl_job.sql.j2 + dest: '{{ install_path }}/{{ admin_soft_home_path }}/xxl_job.sql' + force: yes + run_once: true + delegate_to: '{{groups.services[0]}}' + +- name: copy {{ admin_image_name }} create_mariadb_tables.sh.j2 + template: + src: create_mariadb_tables.sh.j2 + dest: '{{ install_path }}/{{ admin_soft_home_path }}/create_mariadb_tables.sh' + mode: 0755 + force: yes + run_once: true + delegate_to: '{{groups.services[0]}}' + +- name: create the mysql's tables and insert some datas + shell: cd {{ install_path }}/{{ admin_soft_home_path }} && ./create_mariadb_tables.sh + run_once: true + delegate_to: '{{groups.services[0]}}' diff --git a/parcels/roles/services/galaxy-job-service/tasks/load_job.yml b/parcels/roles/services/galaxy-job-service/tasks/load_job.yml new file mode 100644 index 0000000..aa96ca9 --- /dev/null +++ b/parcels/roles/services/galaxy-job-service/tasks/load_job.yml @@ -0,0 +1,111 @@ +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181" + when: '(groups.zookeeper|length) == 3' + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181,{{groups.zookeeper[3]}}:2181,{{groups.zookeeper[4]}}:2181" + when: '(groups.zookeeper|length) >= 5' + +- name: get job_admin_servers to ansible variable + set_fact: job_admin_servers="http://{{ groups.services[0] }}:8184/xxl-job-admin,http://{{ groups.services[1] }}:8184/xxl-job-admin" + when: '(groups.services|length) == 2' + +- name: get job_admin_servers to ansible variable + set_fact: job_admin_servers="http://{{ groups.services[0] }}:8184/xxl-job-admin,http://{{ groups.services[1] }}:8184/xxl-job-admin,http://{{ groups.services[2] }}:8184/xxl-job-admin" + when: '(groups.services|length) == 3' + +- name: stop and remove {{ job_image_container_name }} container + docker_container: + name: '{{ job_image_container_name }}' + state: absent + +- name: remove old {{ job_image_name }} image + docker_image: + name: '{{ job_image_name }}' + tag: '{{ job_image_tag_name }}' + state: absent + +- name: create {{ job_volume_path }} + file: + state: directory + path: '{{ install_path }}/{{ job_volume_path }}/config' + +- name: create {{ job_soft_home_path }} + file: + state: directory + path: '{{ install_path }}/{{ job_soft_home_path }}' + +- name: copy {{ job_image_tar_name }} + copy: + src: '{{ package_path }}/{{ job_image_tar_name }}' + dest: '{{ install_path }}/{{ job_soft_home_path }}/{{ job_image_tar_name }}' + force: true + backup: yes + +- name: load new {{ job_image_name }} image from {{ job_image_tar_name }} + docker_image: + name: '{{ job_image_name }}' + tag: '{{ job_image_tag_name }}' + load_path: '{{ install_path }}/{{ job_soft_home_path }}/{{ job_image_tar_name }}' + source: load + force_tag: yes + force_source: yes + +- name: change the image tag + shell: docker tag galaxy-job-executor:{{ job_image_tag_name }} {{ job_image_name }}:{{ job_image_tag_name }} + +- name: copy {{ job_image_name }} docker-compose.yml + template: + src: docker-compose.yml.j2 + dest: '{{ install_path }}/{{ job_soft_home_path }}/docker-compose.yml' + backup: yes + +- name: copy {{ job_image_name }} job_application.properties.j2 + template: + src: job_application.properties.j2 + dest: '{{ install_path }}/{{ job_volume_path }}/config/application.properties' + backup: yes + +#- name: copy {{ job_image_name }} job_application-executor.yml.j2 +# template: +# src: job_application-executor.yml.j2 +# dest: '{{ install_path }}/{{ job_volume_path }}/config/application-executor.yml' +# backup: yes + +- name: copy {{ job_image_name }} logback.xml + template: + src: job_logback.xml.j2 + dest: '{{ install_path }}/{{ job_volume_path }}/config/logback.xml' + backup: yes + +- name: copy config + template: + src: galaxy-job-admin.j2 + dest: '{{ install_path }}/{{ job_soft_home_path }}/galaxy-job-admin' + backup: yes + +- name: copy config + template: + src: galaxy-job-executor.j2 + dest: '{{ install_path }}/{{ job_soft_home_path }}/galaxy-job-executor' + backup: yes + +- name: push config + shell: 'curl --data-urlencode content="`cat {{ install_path }}/{{ job_soft_home_path }}/galaxy-job-executor`" --request POST "http://{{ gateway_keepalive_host }}:8848/nacos/v1/cs/configs?username=nacos&password={{ nacos_pin }}&tenant={{ services_config_namespace }}&group=Galaxy&dataId=galaxy-job-executor.properties&appName=galaxy-job-executor&type=properties"' + register: change_out + +- fail: + msg: "galaxy-job-executor 配置提交失败" + when: change_out.stdout != 'true' + +- name: push config + shell: 'curl --data-urlencode content="`cat {{ install_path }}/{{ job_soft_home_path }}/galaxy-job-admin`" --request POST "http://{{ gateway_keepalive_host }}:8848/nacos/v1/cs/configs?username=nacos&password={{ nacos_pin }}&tenant={{ services_config_namespace }}&group=Galaxy&dataId=galaxy-job-admin.properties&appName=galaxy-job-admin&type=properties"' + register: change_out + +- fail: + msg: "galaxy-job-admin 配置提交失败" + when: change_out.stdout != 'true' + +- name: start {{ job_image_container_name }} container + docker_compose: + project_src: '{{ install_path }}/{{ job_soft_home_path }}' diff --git a/parcels/roles/services/galaxy-job-service/tasks/main.yml b/parcels/roles/services/galaxy-job-service/tasks/main.yml new file mode 100644 index 0000000..b78d6f5 --- /dev/null +++ b/parcels/roles/services/galaxy-job-service/tasks/main.yml @@ -0,0 +1,4 @@ +#- include: update_python3_and_docker_py.yml +- include: load_admin.yml +- include: load_job.yml + diff --git a/parcels/roles/services/galaxy-job-service/templates/admin_application.properties.j2 b/parcels/roles/services/galaxy-job-service/templates/admin_application.properties.j2 new file mode 100644 index 0000000..001e853 --- /dev/null +++ b/parcels/roles/services/galaxy-job-service/templates/admin_application.properties.j2 @@ -0,0 +1,21 @@ +#配置集的配置格式 +nacos.config.type=properties +#配置中心地址 +nacos.config.server-addr={{ gateway_keepalive_host }}:8848 +#命名空间 +nacos.config.namespace={{ services_config_namespace }} +#数据集ID +nacos.config.data-id=galaxy-job-admin.properties +#开启自动刷新 +nacos.config.auto-refresh=true +#配置对应的分组 +nacos.config.group=Galaxy +#Nacos认证用户 +nacos.config.username=nacos +#Nacos认证密码 +nacos.config.password={{ nacos_pin }} +#开启配置预加载功能 +nacos.config.bootstrap.enable=true +#开启Nacos支持日志级别的加载时机 +nacos.config.bootstrap.log-enable=true + diff --git a/parcels/roles/services/galaxy-job-service/templates/admin_logback.xml.j2 b/parcels/roles/services/galaxy-job-service/templates/admin_logback.xml.j2 new file mode 100644 index 0000000..04aae26 --- /dev/null +++ b/parcels/roles/services/galaxy-job-service/templates/admin_logback.xml.j2 @@ -0,0 +1,45 @@ +<?xml version="1.0" encoding="UTF-8"?> +<configuration scan="true" scanPeriod="60 seconds" debug="false"> + + <!--每100M压缩日志文件--> + <property name="LOG_SIZE" value="100MB"/> + <!--日志文件路径--> + <property name="LOG_PATH" value="/logs"/> + <!--日志文件名称--> + <property name="LOG_FILE_NAME" value="galaxy-job-admin"/> + <!--日志打印等级--> + <property name="LOG_LEVEL" value="info"/> + <!--日志最大的历史30天 --> + <property name="LOG_DAYS" value="30"/> + <!--日志打印格式--> + <property name="LOG_PATTERN" value="%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger -%msg%n"/> + + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>${LOG_PATTERN}</pattern> + </encoder> + </appender> + + <appender name="ALL" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_PATH}/${LOG_FILE_NAME}.log</file> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>ALL</level> + </filter> + <encoder> + <pattern>${LOG_PATTERN}</pattern> + </encoder> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern> + ${LOG_PATH}/history/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz + </fileNamePattern> + <maxHistory>${LOG_DAYS}</maxHistory> + <maxFileSize>${LOG_SIZE}</maxFileSize> + </rollingPolicy> + </appender> + + <root level="${LOG_LEVEL}"> + <appender-ref ref="ALL"/> + <appender-ref ref="STDOUT"/> + </root> + +</configuration>
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-job-service/templates/create_mariadb_tables.sh.j2 b/parcels/roles/services/galaxy-job-service/templates/create_mariadb_tables.sh.j2 new file mode 100644 index 0000000..c072c5d --- /dev/null +++ b/parcels/roles/services/galaxy-job-service/templates/create_mariadb_tables.sh.j2 @@ -0,0 +1,6 @@ +#!/bin/bash + +mysql -uroot -p{{ galaxy_mariadb_pin }} -P3306 -h{{ gateway_keepalive_host }} -e "create database if not exists {{ mariadb_job_database }} default character set utf8mb4 collate utf8mb4_general_ci;" + + +mysql -uroot -p{{ galaxy_mariadb_pin }} -P3306 -h{{ gateway_keepalive_host }} {{ mariadb_job_database }} < xxl_job.sql diff --git a/parcels/roles/services/galaxy-job-service/templates/docker-compose.yml.j2 b/parcels/roles/services/galaxy-job-service/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..3a13740 --- /dev/null +++ b/parcels/roles/services/galaxy-job-service/templates/docker-compose.yml.j2 @@ -0,0 +1,32 @@ +version: '2' + +services: + galaxy-job-executor: + image: {{ job_image_name }}:{{ job_image_tag_name }} + container_name: galaxy-job + environment: + JAVA_OPTS: "{{ job_java_opts }}" + ports: + - "8182:8182" + volumes: + - "{{ install_path }}/{{ job_volume_path }}/data:/data" + - "{{ install_path }}/{{ job_volume_path }}/config:/home/tsg/galaxy/galaxy-job/galaxy-job-executor/config" + - "/etc/localtime:/etc/localtime:ro" + - "/etc/timezone:/etc/timezone:ro" + restart: always + network_mode: "host" + + galaxy-job-admin: + image: {{ admin_image_name }}:{{ admin_image_tag_name }} + container_name: xxl-job-admin + environment: + JAVA_OPTS: "{{ job_java_opts }}" + ports: + - "8181:8181" + volumes: + - "{{ install_path }}/{{ admin_volume_path }}/data:/data" + - "{{ install_path }}/{{ admin_volume_path }}/config:/home/tsg/galaxy/galaxy-job/galaxy-job-admin/config" + - "/etc/localtime:/etc/localtime:ro" + - "/etc/timezone:/etc/timezone:ro" + restart: always + network_mode: "host" diff --git a/parcels/roles/services/galaxy-job-service/templates/galaxy-job-admin.j2 b/parcels/roles/services/galaxy-job-service/templates/galaxy-job-admin.j2 new file mode 100644 index 0000000..b0f70c5 --- /dev/null +++ b/parcels/roles/services/galaxy-job-service/templates/galaxy-job-admin.j2 @@ -0,0 +1,73 @@ +### web +server.port=8184 +server.servlet.context-path=/xxl-job-admin +spring.application.name=galaxy-job-admin +### actuator +management.server.servlet.context-path=/actuator +management.health.mail.enabled=false +management.endpoints.web.exposure.include=* +#详细的应用健康信息 prometheus +management.endpoint.health.show-details=always + +management.endpoint.metrics.enabled=true +management.endpoint.prometheus.enabled=true +management.metrics.export.prometheus.enabled=true +management.metrics.tags.application=${spring.application.name} +management.metrics.tags.module=${spring.application.name} + + +### resources +spring.mvc.servlet.load-on-startup=0 +spring.mvc.static-path-pattern=/static/** +spring.resources.static-locations=classpath:/static/ + +### freemarker +spring.freemarker.templateLoaderPath=classpath:/templates/ +spring.freemarker.suffix=.ftl +spring.freemarker.charset=UTF-8 +spring.freemarker.request-context-attribute=request +spring.freemarker.settings.number_format=0.########## + +### mybatis +mybatis.mapper-locations=classpath:/mybatis-mapper/*Mapper.xml +#mybatis.type-aliases-package=com.xxl.job.admin.core.model + +### xxl-job, datasource +spring.datasource.url=jdbc:mysql://{{ gateway_keepalive_host }}:{{ galaxy_mariadb_port }}/{{ mariadb_job_database }}?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=GMT +spring.datasource.username={{ galaxy_mariadb_username }} +spring.datasource.password={{ galaxy_mariadb_pin }} +spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver + +### datasource-pool +spring.datasource.type=com.zaxxer.hikari.HikariDataSource +spring.datasource.hikari.minimum-idle=10 +spring.datasource.hikari.maximum-pool-size=100 +spring.datasource.hikari.auto-commit=true +spring.datasource.hikari.idle-timeout=30000 +spring.datasource.hikari.pool-name=HikariCP +spring.datasource.hikari.max-lifetime=900000 +spring.datasource.hikari.connection-timeout=30000 +spring.datasource.hikari.connection-test-query=SELECT 1 + +### xxl-job, email +spring.mail.host=smtp.qq.com +spring.mail.port=25 +spring.mail.password=xxx +spring.mail.properties.mail.smtp.auth=true +spring.mail.properties.mail.smtp.starttls.enable=true +spring.mail.properties.mail.smtp.starttls.required=true +spring.mail.properties.mail.smtp.socketFactory.class=javax.net.ssl.SSLSocketFactory + +### xxl-job, access token +xxl.job.accessToken= + +### xxl-job, i18n (default is zh_CN, and you can choose "zh_CN", "zh_TC" and "en") +xxl.job.i18n=zh_CN + +## xxl-job, triggerpool max size +xxl.job.triggerpool.fast.max=200 +xxl.job.triggerpool.slow.max=100 + +### xxl-job, log retention days +xxl.job.logretentiondays=30 diff --git a/parcels/roles/services/galaxy-job-service/templates/galaxy-job-executor.j2 b/parcels/roles/services/galaxy-job-service/templates/galaxy-job-executor.j2 new file mode 100644 index 0000000..c079a0e --- /dev/null +++ b/parcels/roles/services/galaxy-job-service/templates/galaxy-job-executor.j2 @@ -0,0 +1,62 @@ +################################静态参数配置(修改后需要重启项目)################################ +### web port +server.port = 8185 +spring.application.name=galaxy-job-executor +### actuator +management.server.servlet.context-path=/actuator +management.health.mail.enabled=false +management.endpoints.web.exposure.include=* +#详细的应用健康信息 +management.endpoint.health.show-details=always +management.endpoint.metrics.enabled=true +management.endpoint.prometheus.enabled=true +management.metrics.export.prometheus.enabled=true +management.metrics.tags.application=${spring.application.name} +zookeeper.server={{ zookeeper_servers }} + +################################动态参数配置(修改后不需要重启项目)################################ +##存储配额文件服务器 +storge.files.hos-server=cityA|{{ hos_keepalive_host }}:9098 +storge.files.token=c21f969b5f03d33d43e04f8f136e7682 +##存储配额查询druid +storge.analytic.server=cityA|{{ gateway_keepalive_host }}:8089 +##存储配额查询clickhouse +storge.traffic.server=cityA|{{ gateway_keepalive_host }}:8123 +storge.traffic.datasource=tsg_galaxy_v3 +storge.traffic.username=default +storge.traffic.password=ceiec2019 +#删除ttl +storge.traffic.system.parts=system.parts +#存储配额查询 +storge.traffic.system.partsclusters=system.parts_cluster +storge.traffic.system.disks=system.disks_cluster +storge.traffic.system.tables=system.tables +storge.traffic.system.clusters=system.clusters +#删除ttl白名单,多个逗号分隔 +storge.files.delete.exclusion= +storge.analytic.delete.exclusion=traffic_metrics_log +storge.taffic.delete.exclusion= +### xxl-job admin address list, such as "http://address" or "http://address01,http://address02" +xxl.job.admin.addresses={{ job_admin_servers }} +### xxl-job, access token +xxl.job.accessToken= +### xxl-job executor registry-address: default use address to registry , otherwise use ip:po +xxl.job.executor.appname=galaxy-executor +### xxl-job executor registry-address: default use address to registry , otherwise use ip:port if address is null +xxl.job.executor.address= +### xxl-job executor server-info +xxl.job.executor.ip= +xxl.job.executor.port=8886 +### xxl-job executor log-path +xxl.job.executor.logpath=/data/logs/jobhandler +### xxl-job executor log-retention-days +xxl.job.executor.logretentiondays=30 +## http pool config +### max connection number +http.pool.max.connection=500 +http.pool.request.timeout=120000 +http.pool.response.timeout=120000 +http.pool.max.per.route=300 +http.pool.connect.timeout=10000 +##指定kafka server的地址，集群配多个，中间，逗号隔开 +spring.kafka.bootstrap-servers={{ producer_kafka_servers }} diff --git a/parcels/roles/services/galaxy-job-service/templates/job_application.properties.j2 b/parcels/roles/services/galaxy-job-service/templates/job_application.properties.j2 new file mode 100644 index 0000000..ddda806 --- /dev/null +++ b/parcels/roles/services/galaxy-job-service/templates/job_application.properties.j2 @@ -0,0 +1,23 @@ +#配置集的配置格式 +nacos.config.type=properties +#配置中心地址 +nacos.config.server-addr={{ gateway_keepalive_host }}:8848 +#命名空间 +nacos.config.namespace={{ services_config_namespace }} +#数据集ID +nacos.config.data-id=galaxy-job-executor.properties +#开启自动刷新 +nacos.config.auto-refresh=true +#配置对应的分组 +nacos.config.group=Galaxy +#Nacos认证用户 +nacos.config.username=nacos +#Nacos认证密码 +nacos.config.password={{ nacos_pin }} +#开启配置预加载功能 +nacos.config.bootstrap.enable=true +#开启Nacos支持日志级别的加载时机 +nacos.config.bootstrap.log-enable=true + +### log config +logging.config=classpath:logback.xml diff --git a/parcels/roles/services/galaxy-job-service/templates/job_logback.xml.j2 b/parcels/roles/services/galaxy-job-service/templates/job_logback.xml.j2 new file mode 100644 index 0000000..dc0a996 --- /dev/null +++ b/parcels/roles/services/galaxy-job-service/templates/job_logback.xml.j2 @@ -0,0 +1,45 @@ +<?xml version="1.0" encoding="UTF-8"?> +<configuration scan="true" scanPeriod="60 seconds" debug="false"> + + <!--每100M压缩日志文件--> + <property name="LOG_SIZE" value="100MB"/> + <!--日志文件路径--> + <property name="LOG_PATH" value="/logs"/> + <!--日志文件名称--> + <property name="LOG_FILE_NAME" value="galaxy-job-executor"/> + <!--日志打印等级--> + <property name="LOG_LEVEL" value="info"/> + <!--日志最大的历史30天 --> + <property name="LOG_DAYS" value="30"/> + <!--日志打印格式--> + <property name="LOG_PATTERN" value="%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger -%msg%n"/> + + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>${LOG_PATTERN}</pattern> + </encoder> + </appender> + + <appender name="ALL" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${LOG_PATH}/${LOG_FILE_NAME}.log</file> + <filter class="ch.qos.logback.classic.filter.LevelFilter"> + <level>ALL</level> + </filter> + <encoder> + <pattern>${LOG_PATTERN}</pattern> + </encoder> + <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> + <fileNamePattern> + ${LOG_PATH}/history/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz + </fileNamePattern> + <maxHistory>${LOG_DAYS}</maxHistory> + <maxFileSize>${LOG_SIZE}</maxFileSize> + </rollingPolicy> + </appender> + + <root level="${LOG_LEVEL}"> + <appender-ref ref="ALL"/> + <appender-ref ref="STDOUT"/> + </root> + +</configuration>
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-job-service/templates/xxl_job.sql.j2 b/parcels/roles/services/galaxy-job-service/templates/xxl_job.sql.j2 new file mode 100644 index 0000000..257efaf --- /dev/null +++ b/parcels/roles/services/galaxy-job-service/templates/xxl_job.sql.j2 @@ -0,0 +1,193 @@ +/* +Navicat MySQL Data Transfer + +Source Server : 192.168.44.3 +Source Server Version : 50505 +Source Host : 192.168.44.3:3306 +Source Database : xxl_job + +Target Server Type : MYSQL +Target Server Version : 50505 +File Encoding : 65001 + +Date: 2020-10-19 10:28:42 +*/ + +SET FOREIGN_KEY_CHECKS=0; + +-- ---------------------------- +-- Table structure for `xxl_job_group` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_group`; +CREATE TABLE `xxl_job_group` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `app_name` varchar(64) NOT NULL COMMENT '执行器AppName', + `title` varchar(12) NOT NULL COMMENT '执行器名称', + `address_type` tinyint(4) NOT NULL DEFAULT 0 COMMENT '执行器地址类型：0=自动注册、1=手动录入', + `address_list` varchar(512) DEFAULT NULL COMMENT '执行器地址列表，多地址逗号分隔', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- +-- Records of xxl_job_group +-- ---------------------------- +INSERT INTO `xxl_job_group` VALUES ('3', 'galaxy-executor', '数据平台业务执行器', '1', '{{ job_executor_servers }}'); + +-- ---------------------------- +-- Table structure for `xxl_job_info` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_info`; +CREATE TABLE `xxl_job_info` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `job_group` int(11) NOT NULL COMMENT '执行器主键ID', + `job_cron` varchar(128) NOT NULL COMMENT '任务执行CRON', + `job_desc` varchar(255) NOT NULL, + `add_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `author` varchar(64) DEFAULT NULL COMMENT '作者', + `alarm_email` varchar(255) DEFAULT NULL COMMENT '报警邮件', + `executor_route_strategy` varchar(50) DEFAULT NULL COMMENT '执行器路由策略', + `executor_handler` varchar(255) DEFAULT NULL COMMENT '执行器任务handler', + `executor_param` varchar(4000) DEFAULT '' COMMENT '执行器任务参数', + `executor_block_strategy` varchar(50) DEFAULT NULL COMMENT '阻塞处理策略', + `executor_timeout` int(11) NOT NULL DEFAULT 0 COMMENT '任务执行超时时间，单位秒', + `executor_fail_retry_count` int(11) NOT NULL DEFAULT 0 COMMENT '失败重试次数', + `glue_type` varchar(50) NOT NULL COMMENT 'GLUE类型', + `glue_source` mediumtext DEFAULT NULL COMMENT 'GLUE源代码', + `glue_remark` varchar(128) DEFAULT NULL COMMENT 'GLUE备注', + `glue_updatetime` datetime DEFAULT NULL COMMENT 'GLUE更新时间', + `child_jobid` varchar(255) DEFAULT NULL COMMENT '子任务ID，多个逗号分隔', + `trigger_status` tinyint(4) NOT NULL DEFAULT 0 COMMENT '调度状态：0-停止，1-运行', + `trigger_last_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '上次调度时间', + `trigger_next_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '下次调度时间', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=77 DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- +-- Records of xxl_job_info +-- ---------------------------- +INSERT INTO `xxl_job_info` VALUES ('29', '3', '0 0 4 * * ?', '获取CK存储配额-sh(实时压缩负值问题）', '2020-03-26 10:58:09', '2020-08-03 10:04:29', 'zhq', '', 'FAILOVER', 'getStorageQuotaJobHandler', '{\"topic\":\"SYS-STORAGE-LOG\",\"ckDayGrowth\":\"true\"}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-03-26 10:58:09', '', '1', '1603051200000', '1603137600000'); +INSERT INTO `xxl_job_info` VALUES ('36', '3', '0 0/5 * * * ?', '获取所有存储配额-sh', '2020-04-07 21:41:20', '2020-08-03 10:30:05', 'zhq', '', 'FAILOVER', 'getStorageQuotaJobHandler', '{\"topic\":\"SYS-STORAGE-LOG\",\"ckDayGrowth\":\"false\"}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-04-07 21:41:20', '', '1', '1603074600000', '1603074900000'); +INSERT INTO `xxl_job_info` VALUES ('44', '3', '0 0/5 * * * ?', 'Event-Security-Proxy-Top-URLS-By-Hits', '2020-04-26 11:01:44', '2020-10-19 10:20:14', 'galaxy', '', 'FAILOVER', 'httpToKafkaJobHandler', '{\r\n \"url\": \"http://{{ gateway_keepalive_host }}:8089/druid/v2/sql\",\r\n \"method\": \"post\",\r\n \"requestBody\": {\r\n \"query\": \"(SELECT url ,sum( session_num ) AS session_num,TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time FROM security_event_urls_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP, \'PT5M\') GROUP BY url ORDER BY session_num DESC LIMIT 1000) UNION ALL(SELECT url, sum(session_num) AS session_num, TIME_FLOOR(CURRENT_TIMESTAMP, \'PT5M\') as stat_time FROM proxy_event_urls_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP, \'PT5M\'), \'PT5M\', -1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP, \'PT5M\') GROUP BY url ORDER BY session_num DESC LIMIT 1000)\",\r\n \"context\": {\r\n \"skipEmptyBuckets\": \"false\"\r\n },\r\n \"resultFormat\": \"object\"\r\n },\r\n \"resultKey\": \"\",\r\n \"topic\": \"TOP-URLS-LOG\"\r\n}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-04-26 11:01:44', '', '1', '1603074600000', '1603074900000'); +INSERT INTO `xxl_job_info` VALUES ('45', '3', '0 0/5 * * * ?', 'Traffic-Top-Website-Domain', '2020-04-26 11:18:26', '2021-01-02 15:33:03', 'galaxy', '', 'FAILOVER', 'httpToKafkaJobHandler', '{\r\n \"url\": \"http://{{ gateway_keepalive_host }}:8089/druid/v2/sql\",\r\n \"method\": \"post\",\r\n \"requestBody\": {\r\n \"query\": \"(SELECT domain,sum( session_num ) AS session_num,sum( c2s_byte_num ) AS c2s_byte_num,sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num,TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'packets\' as order_by FROM website_domain_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND domain != \'\' GROUP BY domain ORDER BY c2s_pkt_num+s2c_pkt_num DESC LIMIT 1000) UNION ALL (SELECT domain,sum( session_num ) AS session_num,sum( c2s_byte_num ) AS c2s_byte_num,sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num,TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'bytes\' as order_by FROM website_domain_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND domain != \'\' GROUP BY domain ORDER BY c2s_byte_num+s2c_byte_num DESC LIMIT 1000) UNION ALL (SELECT domain,sum(session_num) AS session_num, sum(c2s_byte_num) AS c2s_byte_num, sum(s2c_byte_num) AS s2c_byte_num, sum(c2s_pkt_num) AS c2s_pkt_num,sum(s2c_pkt_num) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'sessions\' as order_by FROM website_domain_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND domain != \'\' GROUP BY domain ORDER BY session_num DESC LIMIT 1000)\",\r\n \"context\": {\r\n \"skipEmptyBuckets\": \"false\"\r\n },\r\n \"resultFormat\": \"object\"\r\n },\r\n \"resultKey\": \"\",\r\n \"topic\": \"TOP-WEBSITE-DOMAIN-LOG\"\r\n}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-04-26 11:18:26', '', '1', '1609739400000', '1609739700000'); +INSERT INTO `xxl_job_info` VALUES ('46', '3', '0 0/5 * * * ?', 'Traffic-Top-User', '2020-04-26 11:23:08', '2021-01-02 15:32:34', 'galaxy', '', 'FAILOVER', 'httpToKafkaJobHandler', '{\r\n \"url\": \"http://{{ gateway_keepalive_host }}:8089/druid/v2/sql\",\r\n \"method\": \"post\",\r\n \"requestBody\": {\r\n \"query\": \"(SELECT common_subscriber_id as subscriber_id, sum( session_num ) AS session_num,sum( c2s_byte_num ) AS c2s_byte_num,sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num, sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'packets\' as order_by FROM user_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_subscriber_id != \'\' GROUP BY common_subscriber_id ORDER BY c2s_pkt_num+s2c_pkt_num DESC LIMIT 1000) UNION ALL (SELECT common_subscriber_id as subscriber_id,sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num, sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'bytes\' as order_by FROM user_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_subscriber_id != \'\' GROUP BY common_subscriber_id ORDER BY c2s_byte_num+s2c_byte_num DESC LIMIT 1000) UNION ALL (SELECT common_subscriber_id as subscriber_id,sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num, sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num,TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'sessions\' as order_by FROM user_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_subscriber_id != \'\' GROUP BY common_subscriber_id ORDER BY session_num DESC LIMIT 1000)\",\r\n \"context\": {\r\n \"skipEmptyBuckets\": \"false\"\r\n },\r\n \"resultFormat\": \"object\"\r\n },\r\n \"resultKey\": \"\",\r\n \"topic\": \"TOP-USER-LOG\"\r\n}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-04-26 11:23:08', '', '1', '1609739400000', '1609739700000'); +INSERT INTO `xxl_job_info` VALUES ('47', '3', '0 0/5 * * * ?', 'Traffic-Top-External-Host', '2020-04-26 11:32:41', '2021-01-02 15:32:02', 'galaxy', '', 'FAILOVER', 'httpToKafkaJobHandler', '{\r\n \"url\": \"http://{{ gateway_keepalive_host }}:8089/druid/v2/sql\",\r\n \"method\": \"post\",\r\n \"requestBody\": {\r\n \"query\": \"(SELECT common_external_ip AS destination, sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num,TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'packets\' as order_by FROM server_external_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_external_ip != \'\' GROUP BY common_external_ip ORDER BY c2s_pkt_num+s2c_pkt_num DESC LIMIT 1000) UNION ALL (SELECT common_external_ip AS destination, sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'bytes\' as order_by FROM server_external_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_external_ip != \'\' GROUP BY common_external_ip ORDER BY c2s_byte_num+s2c_byte_num DESC LIMIT 1000) UNION ALL (SELECT common_external_ip AS destination, sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num, sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'sessions\' as order_by FROM server_external_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_external_ip != \'\' GROUP BY common_external_ip ORDER BY session_num DESC LIMIT 1000)\",\r\n \"context\": {\r\n \"skipEmptyBuckets\": \"false\"\r\n },\r\n \"resultFormat\": \"object\"\r\n },\r\n \"resultKey\": \"\",\r\n \"topic\": \"TOP-EXTERNAL-HOST-LOG\"\r\n}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-04-26 11:32:41', '', '1', '1609739400000', '1609739700000'); +INSERT INTO `xxl_job_info` VALUES ('48', '3', '0 0/5 * * * ?', 'Traffic-Top-Internal-Host', '2020-04-26 11:38:57', '2021-01-02 15:31:13', 'galaxy', '', 'FAILOVER', 'httpToKafkaJobHandler', '{\r\n \"url\": \"http://{{ gateway_keepalive_host }}:8089/druid/v2/sql\",\r\n \"method\": \"post\",\r\n \"requestBody\": {\r\n \"query\": \"(SELECT common_internal_ip AS source, sum( session_num ) AS session_num,sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num, sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'packets\' as order_by FROM client_internal_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_internal_ip != \'\' GROUP BY common_internal_ip ORDER BY c2s_pkt_num+s2c_pkt_num DESC LIMIT 1000) UNION ALL (SELECT common_internal_ip AS source, sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num, sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num,TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'bytes\' as order_by FROM client_internal_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_internal_ip != \'\' GROUP BY common_internal_ip ORDER BY c2s_byte_num+s2c_byte_num DESC LIMIT 1000) UNION ALL (SELECT common_internal_ip AS source, sum( session_num ) AS session_num,sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num, sum( c2s_pkt_num ) AS c2s_pkt_num, sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time, \'sessions\' as order_by FROM client_internal_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND common_internal_ip != \'\' GROUP BY common_internal_ip ORDER BY session_num DESC LIMIT 1000)\",\r\n \"context\": {\r\n \"skipEmptyBuckets\": \"false\"\r\n },\r\n \"resultFormat\": \"object\"\r\n },\r\n \"resultKey\": \"\",\r\n \"topic\": \"TOP-INTERNAL-HOST-LOG\"\r\n}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-04-26 11:38:57', '', '1', '1609739400000', '1609739700000'); +INSERT INTO `xxl_job_info` VALUES ('58', '3', '0 0 0 1/1 * ? ', '设置统计数据存储配额( Druid 保留天)', '2020-07-30 10:34:34', '2020-10-15 18:27:29', 'zhq', '', 'FAILOVER', 'deleteReportAndMetricsDataJobHandler', '{\"maxdays\":365}', 'DISCARD_LATER', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-07-30 10:34:34', '', '1', '1603036800000', '1603123200000'); +INSERT INTO `xxl_job_info` VALUES ('59', '3', '0 0 0 1 * ?', '删除所有统计数据(Druid)', '2020-07-30 11:32:08', '2020-10-15 17:13:36', 'zhq', '', 'FAILOVER', 'deleteAllReportAndMetricsDataJobHandler', '{\"maxdays\":365}', 'DISCARD_LATER', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:32:08', '', '0', '0', '0'); +INSERT INTO `xxl_job_info` VALUES ('62', '3', '0 0 0 1/1 * ? ', '设置文件存储配额( hos 保留天)', '2020-07-30 11:59:30', '2020-10-15 17:17:44', 'zhq', '', 'FAILOVER', 'deleteFilesJobHandler', '{\"maxdays\":365}', 'DISCARD_LATER', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:59:30', '', '1', '1603036800000', '1603123200000'); +INSERT INTO `xxl_job_info` VALUES ('63', '3', '0 0 0 1 * ?', '删除所有文件(hos)', '2020-07-30 11:59:43', '2020-10-15 17:17:44', 'zhq', '', 'FAILOVER', 'deleteAllFilesJobHandler', '{\"maxdays\":365}', 'DISCARD_LATER', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:59:43', '', '0', '0', '0'); +INSERT INTO `xxl_job_info` VALUES ('64', '3', '0 0 0 1 * ?', '删除所有原始日志(ClickHouse)', '2020-07-30 13:47:25', '2020-10-15 18:26:17', 'zhq', '', 'FAILOVER', 'deleteAllTrafficDataJobHandler', '{\"maxdays\":29}', 'DISCARD_LATER', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-07-30 13:47:25', '', '0', '0', '0'); +INSERT INTO `xxl_job_info` VALUES ('65', '3', '0 0 0 1/1 * ? ', '设置原始日志存储配额（ClickHouse保留天）', '2020-07-30 13:47:38', '2020-10-15 18:27:08', 'zhq', '', 'FAILOVER', 'deleteTrafficDataJobHandler', '{\"maxdays\":30}', 'DISCARD_LATER', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-07-30 13:47:38', '', '1', '1603036800000', '1603123200000'); +INSERT INTO `xxl_job_info` VALUES ('66', '3', '0 0/5 * * * ?', 'Traffic-Top-Client-Ip', '2020-09-23 14:43:56', '2021-01-02 15:30:48', 'galaxy', '', 'FAILOVER', 'httpToKafkaJobHandler', '{\r\n \"url\": \"http://{{ gateway_keepalive_host }}:8089/druid/v2/sql\",\r\n \"method\": \"post\",\r\n \"requestBody\": {\r\n \"query\": \"(SELECT common_client_ip AS source, sum( session_num ) AS session_num,sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num, sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'packets\' as order_by FROM client_internal_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND (common_l4_protocol = \'IPv6_TCP\' OR common_l4_protocol = \'IPv4_TCP\') GROUP BY common_client_ip ORDER BY c2s_pkt_num+s2c_pkt_num DESC LIMIT 1000) UNION ALL (SELECT common_client_ip AS source, sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num, sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num,TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'bytes\' as order_by FROM client_internal_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND (common_l4_protocol = \'IPv6_TCP\' OR common_l4_protocol = \'IPv4_TCP\') GROUP BY common_client_ip ORDER BY c2s_byte_num+s2c_byte_num DESC LIMIT 1000) UNION ALL (SELECT common_client_ip AS source, sum( session_num ) AS session_num,sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num, sum( c2s_pkt_num ) AS c2s_pkt_num, sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time, \'sessions\' as order_by FROM client_internal_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND (common_l4_protocol = \'IPv6_TCP\' OR common_l4_protocol = \'IPv4_TCP\') GROUP BY common_client_ip ORDER BY session_num DESC LIMIT 1000)\",\r\n \"context\": {\r\n \"skipEmptyBuckets\": \"false\"\r\n },\r\n \"resultFormat\": \"object\"\r\n },\r\n \"resultKey\": \"\",\r\n \"topic\": \"TOP-CLIENT-IP-LOG\"\r\n}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-09-23 14:43:56', '', '1', '1609739400000', '1609739700000'); +INSERT INTO `xxl_job_info` VALUES ('67', '3', '0 0/5 * * * ?', 'Traffic-Top-Server-Ip', '2020-09-23 14:46:37', '2021-01-02 15:30:21', 'galaxy', '', 'FAILOVER', 'httpToKafkaJobHandler', '{\r\n \"url\": \"http://{{ gateway_keepalive_host }}:8089/druid/v2/sql\",\r\n \"method\": \"post\",\r\n \"requestBody\": {\r\n \"query\": \"(SELECT common_server_ip AS destination, sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num,TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'packets\' as order_by FROM server_external_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND (common_l4_protocol = \'IPv6_TCP\' OR common_l4_protocol = \'IPv4_TCP\') GROUP BY common_server_ip ORDER BY c2s_pkt_num+s2c_pkt_num DESC LIMIT 1000) UNION ALL (SELECT common_server_ip AS destination, sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num,sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'bytes\' as order_by FROM server_external_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND (common_l4_protocol = \'IPv6_TCP\' OR common_l4_protocol = \'IPv4_TCP\') GROUP BY common_server_ip ORDER BY c2s_byte_num+s2c_byte_num DESC LIMIT 1000) UNION ALL (SELECT common_server_ip AS destination, sum( session_num ) AS session_num, sum( c2s_byte_num ) AS c2s_byte_num, sum( s2c_byte_num ) AS s2c_byte_num,sum( c2s_pkt_num ) AS c2s_pkt_num, sum( s2c_pkt_num ) AS s2c_pkt_num, TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') as stat_time,\'sessions\' as order_by FROM server_external_ip_hot_log WHERE __time >= TIME_SHIFT(TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\'),\'PT5M\',-1) AND __time < TIME_FLOOR(CURRENT_TIMESTAMP,\'PT5M\') AND (common_l4_protocol = \'IPv6_TCP\' OR common_l4_protocol = \'IPv4_TCP\') GROUP BY common_server_ip ORDER BY session_num DESC LIMIT 1000)\",\r\n \"context\": {\r\n \"skipEmptyBuckets\": \"false\"\r\n },\r\n \"resultFormat\": \"object\"\r\n },\r\n \"resultKey\": \"\",\r\n \"topic\": \"TOP-SERVER-IP-LOG\"\r\n}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2020-09-23 14:46:37', '', '1', '1609739400000', '1609739700000'); +INSERT INTO `xxl_job_info` VALUES ('68', '3', '0 0 0 1/1 * ? ', '设置Druid hot日志保留天', '2020-09-25 19:29:09', '2020-09-28 11:36:17', 'zhq', '', 'FAILOVER', 'deleteDruidHotLogJobHandler', '{\"maxdays\":1}', 'DISCARD_LATER', '0', '2', 'BEAN', '', 'GLUE代码初始化', '2020-09-25 19:29:09', '', '1', '1603036800000', '1603123200000'); + +-- ---------------------------- +-- Table structure for `xxl_job_lock` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_lock`; +CREATE TABLE `xxl_job_lock` ( + `lock_name` varchar(50) NOT NULL COMMENT '锁名称', + PRIMARY KEY (`lock_name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- +-- Records of xxl_job_lock +-- ---------------------------- +INSERT INTO `xxl_job_lock` VALUES ('schedule_lock'); + +-- ---------------------------- +-- Table structure for `xxl_job_log` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_log`; +CREATE TABLE `xxl_job_log` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `job_group` int(11) NOT NULL COMMENT '执行器主键ID', + `job_id` int(11) NOT NULL COMMENT '任务，主键ID', + `executor_address` varchar(255) DEFAULT NULL COMMENT '执行器地址，本次执行的地址', + `executor_handler` varchar(255) DEFAULT NULL COMMENT '执行器任务handler', + `executor_param` varchar(4000) DEFAULT '' COMMENT '执行器任务参数', + `executor_sharding_param` varchar(20) DEFAULT NULL COMMENT '执行器任务分片参数，格式如 1/2', + `executor_fail_retry_count` int(11) NOT NULL DEFAULT 0 COMMENT '失败重试次数', + `trigger_time` datetime DEFAULT NULL COMMENT '调度-时间', + `trigger_code` int(11) NOT NULL COMMENT '调度-结果', + `trigger_msg` text DEFAULT NULL COMMENT '调度-日志', + `handle_time` datetime DEFAULT NULL COMMENT '执行-时间', + `handle_code` int(11) NOT NULL COMMENT '执行-状态', + `handle_msg` text DEFAULT NULL COMMENT '执行-日志', + `alarm_status` tinyint(4) NOT NULL DEFAULT 0 COMMENT '告警状态：0-默认、1-无需告警、2-告警成功、3-告警失败', + PRIMARY KEY (`id`), + KEY `I_trigger_time` (`trigger_time`), + KEY `I_handle_code` (`handle_code`) +) ENGINE=InnoDB AUTO_INCREMENT=1715972 DEFAULT CHARSET=utf8mb4; + + +-- ---------------------------- +-- Table structure for `xxl_job_log_report` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_log_report`; +CREATE TABLE `xxl_job_log_report` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `trigger_day` datetime DEFAULT NULL COMMENT '调度-时间', + `running_count` int(11) NOT NULL DEFAULT 0 COMMENT '运行中-日志数量', + `suc_count` int(11) NOT NULL DEFAULT 0 COMMENT '执行成功-日志数量', + `fail_count` int(11) NOT NULL DEFAULT 0 COMMENT '执行失败-日志数量', + PRIMARY KEY (`id`), + UNIQUE KEY `i_trigger_day` (`trigger_day`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=181 DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- + +-- ---------------------------- +-- Table structure for `xxl_job_logglue` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_logglue`; +CREATE TABLE `xxl_job_logglue` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `job_id` int(11) NOT NULL COMMENT '任务，主键ID', + `glue_type` varchar(50) DEFAULT NULL COMMENT 'GLUE类型', + `glue_source` mediumtext DEFAULT NULL COMMENT 'GLUE源代码', + `glue_remark` varchar(128) NOT NULL COMMENT 'GLUE备注', + `add_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- +-- Records of xxl_job_logglue +-- ---------------------------- + +-- ---------------------------- +-- Table structure for `xxl_job_registry` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_registry`; +CREATE TABLE `xxl_job_registry` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `registry_group` varchar(50) NOT NULL, + `registry_key` varchar(255) NOT NULL, + `registry_value` varchar(255) NOT NULL, + `update_time` datetime DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `i_g_k_v` (`registry_group`,`registry_key`,`registry_value`) +) ENGINE=InnoDB AUTO_INCREMENT=235 DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- +-- Table structure for `xxl_job_user` +-- ---------------------------- +DROP TABLE IF EXISTS `xxl_job_user`; +CREATE TABLE `xxl_job_user` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `username` varchar(50) NOT NULL COMMENT '账号', + `password` varchar(50) NOT NULL COMMENT '密码', + `role` tinyint(4) NOT NULL COMMENT '角色：0-普通用户、1-管理员', + `permission` varchar(255) DEFAULT NULL COMMENT '权限：执行器ID列表，多个逗号分割', + PRIMARY KEY (`id`), + UNIQUE KEY `i_username` (`username`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8mb4; + +-- ---------------------------- +-- Records of xxl_job_user +-- ---------------------------- +INSERT INTO `xxl_job_user` VALUES ('1', 'admin', 'fea191a3fdd9f68503f9fa0e8d0293ab', '1', null); +INSERT INTO `xxl_job_user` VALUES ('2', 'query', '44f057733c7a286299a972f6c0c239af', '0', '3'); diff --git a/parcels/roles/services/galaxy-job-service/vars/main.yml b/parcels/roles/services/galaxy-job-service/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/services/galaxy-job-service/vars/main.yml diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/active_defence_event_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/active_defence_event_log.json new file mode 100644 index 0000000..19205b5 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/active_defence_event_log.json @@ -0,0 +1,322 @@ +{ + "type": "record", + "name": "active_defence_event_log", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "schema_query": { + "dimensions": [ + "common_policy_id", + "ad_target_ip", + "ad_cc_target_url" + ], + "metrics": [ + "ad_target_ip", + "ad_sent_byte_num", + "ad_sent_pkt_num", + "ad_cc_initiate_connection_num", + "ad_cc_established_connection_num", + "ad_cc_rejected_connection_num" + ], + "filters": [ + "common_policy_id", + "ad_target_ip", + "ad_target_port", + "ad_protocol", + "common_address_type", + "ad_sent_byte_num", + "ad_sent_pkt_num", + "ad_cc_initiate_connection_num", + "ad_cc_established_connection_num", + "ad_cc_rejected_connection_num" + ] + }, + "schema_type": { + "REFLECTION": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_address_type", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_entrance_id", + "common_user_region", + "ad_method", + "ad_protocol", + "ad_target_ip", + "ad_target_port", + "ad_target_ip_location", + "ad_target_ip_asn", + "ad_reflector_profile_id", + "ad_sent_pkt_num", + "ad_sent_byte_num", + "ad_generate_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "ad_target_ip", + "ad_target_port", + "ad_reflector_profile_id", + "ad_sent_pkt_num", + "ad_sent_byte_num" + ] + }, + "FLOOD": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_address_type", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_entrance_id", + "common_user_region", + "ad_method", + "ad_protocol", + "ad_target_ip", + "ad_target_port", + "ad_target_ip_location", + "ad_target_ip_asn", + "ad_claimed_src_ip_profile_id", + "ad_sent_pkt_num", + "ad_sent_byte_num", + "ad_generate_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "ad_target_ip", + "ad_target_port", + "ad_claimed_src_ip_profile_id", + "ad_protocol" + ] + }, + "CC": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_address_type", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_entrance_id", + "common_user_region", + "ad_method", + "ad_protocol", + "ad_cc_target_url", + "ad_claimed_src_ip_profile_id", + "ad_cc_initiate_connection_num", + "ad_cc_established_connection_num", + "ad_cc_rejected_connection_num", + "ad_generate_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "ad_cc_target_url", + "ad_claimed_src_ip_profile_id", + "ad_protocol" + ] + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "ad_target_ip", + "ad_target_port", + "ad_cc_target_url" + ] + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "ad_target_ip", + "label": "Target IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_ip_country,geo_asn", + "appendTo": "ad_target_ip_location,ad_target_ip_asn" + } + }, + "type": "string" + }, + { + "name": "ad_target_port", + "label": "Target Port", + "type": "int" + }, + { + "name": "ad_cc_target_url", + "label": "Target URL", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ad_target_ip_location", + "label": "Target Location", + "type": "string" + }, + { + "name": "ad_target_ip_asn", + "label": "Target ASN", + "type": "string" + }, + { + "name": "ad_protocol", + "label": "Protocol", + "type": "string" + }, + { + "name": "ad_method", + "label": "Method", + "type": "string" + }, + { + "name": "ad_claimed_src_ip_profile_id", + "label": "Claimed Profile ID", + "type": "int" + }, + { + "name": "ad_reflector_profile_id", + "label": "Reflector Profile ID", + "type": "int" + }, + { + "name": "ad_sent_pkt_num", + "label": "Packets Sent", + "type": "int" + }, + { + "name": "ad_sent_byte_num", + "label": "Bytes Sent", + "type": "int" + }, + { + "name": "ad_cc_initiate_connection_num", + "label": "Initiate Numbers", + "type": "int" + }, + { + "name": "ad_cc_established_connection_num", + "label": "Established Numbers", + "type": "int" + }, + { + "name": "ad_cc_rejected_connection_num", + "label": "Rejected Numbers", + "type": "int" + }, + { + "name": "ad_generate_time", + "label": "Generate Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "int" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/ck-filter.json b/parcels/roles/services/galaxy-qgw-service/files/config/ck-filter.json new file mode 100644 index 0000000..f6f7ad6 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/ck-filter.json @@ -0,0 +1,71 @@ +{ + "version": "1.0", + "name": "ClickHouse-Raw", + "namespace": "ClickHouse", + "filters": [ + { + "name":"@start", + "value": "'2021-01-11 10:00:00'" + }, + { + "name":"@end", + "value": "'2021-01-13 11:00:00'" + }, + { + "name":"@common_filter", + "value": [ + "common_log_id=1153021139190754263", + "common_client_ip='36.189.226.21'", + "common_internal_ip='223.116.37.192'", + "common_server_ip='8.8.8.8'", + "common_external_ip='111.10.53.14'", + "common_client_port=52607", + "common_server_port=443", + "common_c2s_pkt_num>5", + "common_s2c_pkt_num>5", + "common_c2s_byte_num>100", + "common_s2c_byte_num<200", + "common_schema_type='DNS'", + "common_establish_latency_ms>200", + "common_con_duration_ms>10000", + "common_stream_trace_id=1153021139190754263", + "common_tcp_client_isn=2857077935", + "common_tcp_server_isn=0", + "http_domain='microsoft.com'", + "mail_account='[email protected]'", + "mail_subject='test'", + "dns_qname='qbwup.imtt.qq.com'", + "ssl_sni='note.youdao.com'", + "ssl_con_latency_ms>100", + "ssl_ja3_hash='a0e9f5d64349fb13191bc781f81f42e1'", + "common_client_ip='36.189.226.21' and common_server_ip='8.8.8.8'", + "common_server_ip='111.10.53.14' and common_server_port=443", + "mail_account like 'abc@%'", + "http_domain like '%baidu.com%'", + "ssl_sni like '%youdao.com'", + "common_client_ip in ('36.189.226.21','111.10.53.14')", + "common_server_port not in (80,443)", + "notEmpty(http_domain)", + "http_domain not like '%microsoft.com'" + ] + }, + { + "name":"@index_filter", + "value": [ + "common_log_id=1153021139190754263", + "common_client_ip='36.189.226.21'", + "common_server_ip='8.8.8.8'", + "common_sled_ip='%192.168%'", + "common_stream_trace_id=1153021139190754263", + "http_domain='microsoft.com'", + "ssl_sni='note.youdao.com'", + "common_subscriber_id='%test%'", + "http_domain like '%baidu.com%'", + "ssl_sni like '%youdao.com'", + "common_client_ip in ('36.189.226.21','111.10.53.14')", + "notEmpty(http_domain)", + "http_domain not like '%microsoft.com'" + ] + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/ck-queries-template.sql b/parcels/roles/services/galaxy-qgw-service/files/config/ck-queries-template.sql new file mode 100644 index 0000000..c867da7 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/ck-queries-template.sql @@ -0,0 +1,122 @@ +--Q01.Count(1) +select count(1) from connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) +--Q02.All Fields Query (default) +SELECT * FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) LIMIT 30 +--Q03.All Fields Query order by Time desc +SELECT * FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 +--Q04.All Fields Query order by Time asc +SELECT * FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time asc LIMIT 30 +--Q05.All Fields Query by Filter +SELECT * FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @common_filter ORDER BY common_recv_time DESC LIMIT 30 +--Q06.Default Fields Query by Filter +SELECT toDateTime(common_recv_time) AS common_recv_time , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @common_filter ORDER BY common_recv_time DESC LIMIT 30 +--Q07.All Fields Query (sub query by time) +SELECT * FROM connection_record_log AS connection_record_log WHERE toDateTime(common_recv_time) IN ( SELECT toDateTime(common_recv_time) FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 +--Q08.All Fields Query (sub query by log id) +SELECT * FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( SELECT common_log_id FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 +--Q09.Default Field Query (sub query by time) +SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE toDateTime(common_recv_time) IN ( SELECT toDateTime(common_recv_time) FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY common_recv_time DESC LIMIT 30 +--Q10.Default Field Query (sub query by log id) +SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( select common_log_id FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end)) ORDER BY common_recv_time DESC LIMIT 30 +--Q11.Default Field Query by Server IP (sub query by log id with Index Table) +SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( Select common_log_id FROM connection_record_log_common_server_ip AS connection_record_log_common_server_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30 +--Q12.Default Field Query by Client IP (sub query by log id with Index Table) +SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( Select common_log_id FROM connection_record_log_common_client_ip AS connection_record_log_common_client_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30 +--Q13.Default Field Query by Domain (sub query by log id with Index Table) +SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( Select common_log_id FROM connection_record_log_http_domain AS connection_record_log_http_domain WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30 +--Q14.Default Field Query by Subscriber ID (sub query by log id with Index Table) +SELECT toDateTime(common_recv_time) AS common_recv_time_str , common_log_id , common_client_ip , common_client_port , common_server_ip , common_server_port FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( Select common_log_id FROM connection_record_log_common_subscriber_id AS connection_record_log_common_subscriber_id WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time DESC LIMIT 30 +--Q15.All Fields Query by Client IP (sub query by log id with index Table) +SELECT * FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( SELECT common_log_id FROM connection_record_log_common_client_ip AS connection_record_log_common_client_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY toDateTime(common_recv_time) DESC LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30 +--Q16.All Fields Query by Server IP(sub query by log id with index Table) +SELECT * FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( SELECT common_log_id FROM connection_record_log_common_server_ip AS connection_record_log_common_server_ip WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY common_recv_time LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30 +--Q17.All Fields Query by Domain(sub query by log id with index Table) +SELECT * FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( SELECT common_log_id FROM connection_record_log_http_domain AS connection_record_log_http_domain WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY common_recv_time LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30 +--Q18.All Fields Query by Subscriber ID(sub query by log id with index Table) +SELECT * FROM connection_record_log AS connection_record_log WHERE common_log_id IN ( SELECT common_log_id FROM connection_record_log_common_subscriber_id AS connection_record_log_common_subscriber_id WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ORDER BY common_recv_time LIMIT 30 ) AND ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) AND @index_filter ) ORDER BY common_recv_time desc LIMIT 30 +--Q19.Session Logs Sent to Database Trend(Time Grain 5 minute) +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", count(common_log_id) AS "logs" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) ) ) GROUP BY "Receive Time" LIMIT 10000 +--Q20.Traffic Bandwidth Trend(Time Grain 30 second) +SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 30 SECOND)))) AS stat_time, sum(common_c2s_byte_num) AS bytes_sent, sum(common_s2c_byte_num) AS bytes_received, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets, sum(common_sessions) AS sessions FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY stat_time ORDER BY stat_time ASC LIMIT 10000 +--Q21.Log Tend by Type (Time Grain 5 minute) +SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE)))) AS stat_time, common_schema_type AS type, sum(common_sessions) AS sessions, sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, sum(common_c2s_pkt_num + common_s2c_pkt_num) AS packets FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) GROUP BY stat_time, common_schema_type ORDER BY stat_time ASC LIMIT 10000 +--Q22.Traffic Metrics Analytic +SELECT round(sum(common_s2c_byte_num) * 8 / 300,2) AS trafficInBits, round(sum(common_c2s_byte_num) * 8 / 300,2) AS trafficOutBits, round(sum(common_s2c_byte_num + common_c2s_byte_num) * 8 / 300,2) AS trafficTotalBits, round(sum(common_s2c_pkt_num) / 300,2) AS trafficInPackets, round(sum(common_c2s_pkt_num) / 300,2) AS trafficOutPackets, round(sum(common_s2c_pkt_num + common_c2s_pkt_num) / 300,2) AS trafficTotalPackets, round(sum(common_sessions) / 300,2) AS sessions FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toUnixTimestamp(@start) AND common_recv_time < toUnixTimestamp(@end) +--Q23.Traffic Endpoints Metrics Trend(Time Grain 5 minute) +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", uniq(common_internal_ip) AS "Unique Internal IP", uniq(common_external_ip) AS "Unique External IP", uniq(common_subscriber_id) AS "Unique Subscriber ID", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time" LIMIT 10000 +--Q24.Endpoint Unique Num by L4 Protocol +SELECT 'all' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) UNION ALL SELECT 'tcp' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) UNION ALL SELECT 'UDP' AS type, uniq(common_client_ip) AS client_ips, uniq(common_internal_ip) AS internal_ips, uniq(common_server_ip) AS server_ips, uniq(common_external_ip) AS external_ips, uniq(common_subscriber_id) as subscriber_ids FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND common_l4_protocol IN ( 'IPv4_UDP', 'IPv6_UDP' ) +--Q25.One-sided Connection Trend(Time Grain 5 minute) +SELECT toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE)))) AS stat_time, (CASE WHEN common_stream_dir = 1 THEN 'c2s' WHEN common_stream_dir = 2 THEN 's2c' WHEN common_stream_dir = 3 THEN 'double' ELSE 'None' END) AS type, sum(common_sessions) AS sessions FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY stat_time, common_stream_dir ORDER BY stat_time ASC LIMIT 10000 +--Q26. Estimated One-sided Sessions with Bandwidth +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(common_sessions) AS "sessions", sum(if(common_stream_dir <> 3, common_sessions, 0)) AS "one_side_sessions", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", round(one_side_sessions / sessions, 2) AS one_side_percent FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time" LIMIT 10000 +--Q27.Estimated TCP Sequence Gap Loss +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(common_c2s_byte_num + common_s2c_byte_num) AS "bytes", sum(common_c2s_tcp_lostlen + common_s2c_tcp_lostlen) AS "gap_loss_bytes", round(gap_loss_bytes / bytes, 2) AS gap_loss_percent FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) ) GROUP BY "Receive Time" LIMIT 10000 +--Q28.Top30 Server IP by Bytes +SELECT "server_ip" AS "server_ip" , SUM(coalesce("bytes",0)) AS "bytes" , SUM(coalesce("bytes_sent",0)) AS "Sent" , SUM(coalesce("bytes_received",0)) AS "Received" , SUM(coalesce("sessions",0)) AS "sessions" FROM ( SELECT SUM(coalesce(common_c2s_byte_num,0)) AS "bytes_sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "bytes_received" , SUM(common_c2s_byte_num+common_s2c_byte_num) AS "bytes" , SUM(coalesce(common_sessions,0)) AS "sessions" , common_server_ip AS "server_ip" FROM connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( common_server_ip) ) GROUP BY "server_ip" ORDER BY "bytes" desc ) GROUP BY "server_ip" ORDER BY "bytes" desc LIMIT 30 +--Q29.Top30 Client IP by Sessions +SELECT common_client_ip , COUNT(*) AS sessions FROM connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) GROUP BY common_client_ip ORDER BY sessions desc LIMIT 0,30 +--Q30.Top30 TCP Server Ports by Sessions +SELECT "Server Port" AS "Server Port", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_server_port AS "Server Port", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_l4_protocol IN ( 'IPv4_TCP', 'IPv6_TCP' ) ) GROUP BY "Server Port" LIMIT 1048576) GROUP BY "Server Port" ORDER BY "Sessions" DESC LIMIT 30 +--Q31.Top30 Domian by Bytes +SELECT "domain" AS "Website Domain" , SUM(coalesce("bytes",0)) AS "Throughput" FROM ( SELECT SUM(coalesce(common_c2s_byte_num,0)) AS "bytes_sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "bytes_received" , SUM(coalesce(common_c2s_byte_num+common_s2c_byte_num,0)) AS "bytes" , http_domain AS "domain" FROM connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( http_domain) ) GROUP BY "domain" ORDER BY "bytes" desc ) GROUP BY "domain" ORDER BY "Throughput" desc LIMIT 30 +--Q32.Top30 Endpoint Devices by Bandwidth +SELECT "device_id" AS "device_id", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(common_c2s_byte_num + common_s2c_byte_num) AS bytes, common_device_id AS "device_id" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "device_id" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "device_id" ORDER BY "bytes" DESC LIMIT 30 +--Q33.Top30 Domain by Unique Client IP +SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Client IP", 0)) AS "Client IP" FROM (SELECT http_domain AS "Http.Domain", uniq(common_client_ip) AS "Client IP" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Http.Domain" ORDER BY "Client IP" DESC LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Client IP" DESC LIMIT 30 +--Q34.Top100 Most Time Consuming Domains +SELECT "Domain" AS "Domain", avg(coalesce("Avg Establish Latency(ms)", 0)) AS "Avg Establish Latency(ms)" FROM (SELECT http_domain AS "Domain", avg(coalesce(common_establish_latency_ms, 0)) AS "Avg Establish Latency(ms)" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Domain" LIMIT 1048576) GROUP BY "Domain" ORDER BY "Avg Establish Latency(ms)" DESC LIMIT 100 +--Q35.Top30 Sources by Sessions +SELECT "source" AS "source", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT coalesce(nullif(common_subscriber_id, ''), nullif(common_client_ip, '')) AS "source", sum(coalesce(common_sessions, 0)) AS "sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "source" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "source" ORDER BY "sessions" DESC LIMIT 30 +--Q36.Top30 Destinations by Sessions +SELECT "destination" AS "destination", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT coalesce(nullif(http_domain, ''), nullif(common_server_ip, '')) AS "destination", sum(coalesce(common_sessions, 0)) AS "sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "destination" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "destination" ORDER BY "sessions" DESC LIMIT 30 +--Q37.Top30 Destination Regions by Bandwidth +SELECT "server_location" AS "server_location", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT arrayElement(splitByString(',', common_server_location), length(splitByString(',', common_server_location))) AS "server_location", sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "bytes", sum(coalesce(common_sessions, 0)) AS "sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "server_location" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "server_location" ORDER BY "bytes" DESC LIMIT 30 +--Q38.Top30 URLS by Sessions +SELECT "Http URL" AS "Http URL", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_url AS "Http URL", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Http URL" LIMIT 1048576) GROUP BY "Http URL" ORDER BY "Sessions" DESC LIMIT 30 +--Q39.Top30 Destination Transmission APP by Bandwidth +SELECT "server_ip" AS "server_ip", groupUniqArray(coalesce("trans_app", 0)) AS "trans_app", sum(coalesce("bytes", 0)) AS "bytes", sum(coalesce("bytes_sent", 0)) AS "Sent", sum(coalesce("bytes_received", 0)) AS "Received" FROM (SELECT sum(coalesce(common_c2s_byte_num, 0)) AS "bytes_sent", sum(coalesce(common_s2c_byte_num, 0)) AS "bytes_received", sum(common_c2s_byte_num + common_s2c_byte_num) AS "bytes", groupUniqArray(concat(common_l4_protocol, '/', toString(common_server_port))) AS "trans_app", common_server_ip AS "server_ip" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(common_server_ip) ) GROUP BY "server_ip" ORDER BY "bytes" DESC LIMIT 1048576) GROUP BY "server_ip" ORDER BY "bytes" DESC LIMIT 30 +--Q40.Browsing Users by Website domains and Sessions +SELECT "Subscriber ID" AS "Subscriber ID", "Http.Domain" AS "Http.Domain", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT http_domain AS "Http.Domain", common_subscriber_id AS "Subscriber ID", sum(coalesce(common_sessions, 0)) AS "sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) AND notEmpty(common_subscriber_id) ) GROUP BY "Http.Domain", "Subscriber ID" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "Subscriber ID", "Http.Domain" ORDER BY "sessions" DESC LIMIT 10000 +--Q41.Top Domain and Server IP by Bytes Sent +SELECT "Http.Domain" AS "Http.Domain" , "Server IP" AS "Server IP" , SUM(coalesce("Bytes Sent",0)) AS "Bytes Sent" FROM ( SELECT common_server_ip AS "Server IP" , http_domain AS "Http.Domain" , SUM(coalesce(common_c2s_byte_num+common_s2c_byte_num,0)) AS "Bytes" , SUM(coalesce(common_c2s_byte_num,0)) AS "Bytes Sent" , SUM(coalesce(common_s2c_byte_num,0)) AS "Bytes Received" FROM connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty( http_domain) ) GROUP BY "Server IP" , "Http.Domain" ORDER BY "Bytes" desc LIMIT 1048576 ) GROUP BY "Http.Domain" , "Server IP" ORDER BY "Bytes Sent" desc LIMIT 10000 +--Q42.Top30 Website Domains by Client IP and Sessions +SELECT "Http.Domain" AS "Http.Domain", "Client IP" AS "Client IP", sum(coalesce("sessions", 0)) AS "sessions" FROM (SELECT common_client_ip AS "Client IP", http_domain AS "Http.Domain", sum(coalesce(common_sessions, 0)) AS "sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Client IP", "Http.Domain" ORDER BY "sessions" DESC LIMIT 1048576) GROUP BY "Http.Domain", "Client IP" ORDER BY "sessions" DESC LIMIT 10000 +--Q43.Domain is Accessed by Unique Client IP Trend(bytes Time Grain 5 minute) +SELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) AS _time , http_domain AS Domain, COUNT(DISTINCT(common_client_ip)) AS nums FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) AND http_domain IN ( SELECT http_domain FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) GROUP BY http_domain ORDER BY SUM(common_s2c_byte_num+common_c2s_byte_num) DESC LIMIT 5 ) GROUP BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) , http_domain ORDER BY toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),300)*300) DESC LIMIT 10000 +--Q44. Domain is Accessed by Unique Client IP Trend(sessions,Time Grain 5 minute) +SELECT toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))),3600)*3600) AS stat_time , http_domain , uniq (common_client_ip) AS nums FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start)-604800 AND common_recv_time < toDateTime(@end) AND http_domain IN ( SELECT http_domain FROM connection_record_log AS connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND notEmpty(http_domain) GROUP BY http_domain ORDER BY COUNT(*) desc LIMIT 5 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 3600)*3600), http_domain ORDER BY stat_time desc LIMIT 10000 +--Q45.Bandwidth Trend with Device ID(Time Grain 5 minute) +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", common_device_id AS "Device ID", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Receive Time", "Device ID" LIMIT 10000 +--Q46.Internal IP by Sled IP and Sessions +SELECT "Internal IP" AS "Internal IP", "Sled IP" AS "Sled IP", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_sled_ip AS "Sled IP", common_internal_ip AS "Internal IP", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Sled IP", "Internal IP" LIMIT 1048576) GROUP BY "Internal IP", "Sled IP" ORDER BY "Sessions" DESC LIMIT 10000 +--Q47.Bandwidth Trend with Internal IP (Time Grain 5 minute) +SELECT toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL 5 MINUTE))) AS "Receive Time", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes", sum(coalesce(common_c2s_pkt_num + common_s2c_pkt_num, 0)) AS "Packets", sum(coalesce(common_sessions, 0)) AS "New Sessions", sum(coalesce(common_c2s_byte_num, 0)) AS "Bytes Sent", sum(coalesce(common_s2c_byte_num, 0)) AS "Bytes Received", sum(coalesce(common_c2s_pkt_num, 0)) AS "Packets Sent", sum(coalesce(common_s2c_pkt_num, 0)) AS "Packets Received" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) GROUP BY "Receive Time" LIMIT 10000 +--Q48.Top30 Domains Detail with Internal IP +SELECT "Domain" AS "Domain", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_domain AS "Domain", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) AND ( notEmpty(http_domain) ) GROUP BY "Domain" LIMIT 1048576) GROUP BY "Domain" ORDER BY "Sessions" DESC LIMIT 30 +--Q49.Top30 URLS Detail with Internal IP +SELECT "URL" AS "URL", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT http_url AS "URL", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) AND @common_filter ) AND ( notEmpty(http_url) ) GROUP BY "URL" LIMIT 1048576) GROUP BY "URL" ORDER BY "Sessions" DESC LIMIT 30 +--Q50.Top Domains with Unique Client IP and Subscriber ID +SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Unique Client IP", 0)) AS "Unique Client IP", sum(coalesce("Unique Subscriber ID", 0)) AS "Unique Subscriber ID" FROM (SELECT http_domain AS "Http.Domain", uniq(common_client_ip) AS "Unique Client IP", uniq(common_subscriber_id) AS "Unique Subscriber ID" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( notEmpty(http_domain) ) GROUP BY "Http.Domain" LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Unique Client IP" DESC LIMIT 100 +--Q51.Top100 Domains by Packets sent +SELECT "Http.Domain" AS "Http.Domain", sum(coalesce("Packets Sent", 0)) AS "Packets Sent" FROM (SELECT http_domain AS "Http.Domain", sum(coalesce(common_c2s_pkt_num, 0)) AS "Packets Sent" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Http.Domain" LIMIT 1048576) GROUP BY "Http.Domain" ORDER BY "Packets Sent" DESC LIMIT 100 +--Q52.Internal and External asymmetric traffic +SELECT "Internal IP" AS "Internal IP", "External IP" AS "External IP", "Sled IP" AS "Sled IP", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_sled_ip AS "Sled IP", common_external_ip AS "External IP", common_internal_ip AS "Internal IP", sum(coalesce(common_c2s_byte_num + common_s2c_byte_num, 0)) AS "Bytes Sent+Bytes Received", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_stream_dir != 3 ) GROUP BY "Sled IP", "External IP", "Internal IP" LIMIT 1048576) GROUP BY "Internal IP", "External IP", "Sled IP" ORDER BY "Sessions" DESC LIMIT 500 +--Q53.Client and Server ASN asymmetric traffic +SELECT "Client ASN" AS "Client ASN", "Server ASN" AS "Server ASN", sum(coalesce("Sessions", 0)) AS "Sessions" FROM (SELECT common_server_asn AS "Server ASN", common_client_asn AS "Client ASN", sum(coalesce(common_sessions, 0)) AS "Sessions" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) AND ( common_stream_dir != 3 ) GROUP BY "Server ASN", "Client ASN" LIMIT 1048576) GROUP BY "Client ASN", "Server ASN" ORDER BY "Sessions" DESC LIMIT 500 +--Q54.Top handshake latency by Website and Client IPs +SELECT "SSL.SNI" AS "SSL.SNI", "Client IP" AS "Client IP", avg(coalesce("Establish Latency(ms)", 0)) AS "Establish Latency(ms)" FROM (SELECT common_client_ip AS "Client IP", ssl_sni AS "SSL.SNI", avg(coalesce(common_establish_latency_ms, 0)) AS "Establish Latency(ms)" FROM connection_record_log AS connection_record_log WHERE ( ( common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) ) ) GROUP BY "Client IP", "SSL.SNI" LIMIT 1048576) GROUP BY "SSL.SNI", "Client IP" ORDER BY "Establish Latency(ms)" DESC LIMIT 500 +--Q55.Domain Baidu.com Metrics +select FROM_UNIXTIME(min(common_recv_time)) as "First Seen" , FROM_UNIXTIME(max(common_recv_time)) as "Last Seen" , median(http_response_latency_ms) as "Server Processing Time Median(ms)", count(1) as Responses,any(common_server_location) as Location from connection_record_log WHERE common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) AND http_domain='baidu.com' +--Q56.Domain baidu.com Drill down Client IP +select common_client_ip as "Client IP" , avg(common_establish_latency_ms) as "Establishing Time Mean(ms)", count(1) as Responses,any(common_client_location) as Location from connection_record_log where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and http_domain='baidu.com' group by "Client IP" order by Responses desc limit 100 +--Q57.Domain baidu.com Drill down Server IP +select common_server_ip as "Server IP" , avg(http_response_latency_ms) as "Server Processing Time Mean(ms)", count(1) as Responses,any(common_server_location) as Location from connection_record_log where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and http_domain='baidu.com' group by "Server IP" order by Responses desc limit 100 +--Q58.Domain baidu.com Drill down URI +select http_url as "URI" , avg(http_response_latency_ms) as "Server Processing Time Mean(ms)", count(1) as Responses from connection_record_log where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and http_domain='baidu.com' group by "URI" order by Responses desc limit 100 +--Q59.L7 Protocol Metrics +select common_l7_protocol as "Protocol" , uniq(common_client_ip) as "Clients" , uniq(common_server_ip) as "Servers", count(1) as Sessions,sum(common_c2s_byte_num+common_s2c_byte_num) as bytes from connection_record_log where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and notEmpty(common_l7_protocol) group by common_l7_protocol order by bytes desc +--Q60.L7 Protocol SIP Drill down Client IP +select common_client_ip as "Client IP" , count(1) as Sessions,sum(common_c2s_byte_num) as "Bytes Out", sum(common_s2c_byte_num) as "Bytes In",any(common_client_location) as Location from connection_record_log where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and common_l7_protocol='SIP' group by "Client IP" order by Sessions desc limit 100 +--Q61.L7 Protocol SIP Drill down Server IP +select common_server_ip as "Server IP" , count(1) as Sessions,sum(common_c2s_byte_num) as "Bytes Out", sum(common_s2c_byte_num) as "Bytes In",any(common_server_location) as Location from connection_record_log where common_recv_time >= toDateTime(@start) AND common_recv_time < toDateTime(@end) and common_l7_protocol='SIP' group by "Server IP" order by Sessions desc limit 100
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/columns_cluster.json b/parcels/roles/services/galaxy-qgw-service/files/config/columns_cluster.json new file mode 100644 index 0000000..d190d3c --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/columns_cluster.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "columns_cluster", + "fields": [ + { + "name": "database", + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log.json new file mode 100644 index 0000000..795b433 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log.json @@ -0,0 +1,3626 @@ +{ + "type": "record", + "name": "connection_record_log", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "index_table": "connection_record_log_common_client_ip,connection_record_log_common_server_ip,connection_record_log_common_subscriber_id,connection_record_log_http_domain", + "functions": { + "aggregation": [ + { + "name": "COUNT", + "label": "COUNT", + "function": "count(expr)" + }, + { + "name": "COUNT_DISTINCT", + "label": "COUNT_DISTINCT", + "function": "count(distinct expr)" + }, + { + "name": "AVG", + "label": "AVG", + "function": "avg(expr)" + }, + { + "name": "SUM", + "label": "SUM", + "function": "sum(expr)" + }, + { + "name": "MAX", + "label": "MAX", + "function": "max(expr)" + }, + { + "name": "MIN", + "label": "MIN", + "function": "min(expr)" + } + ], + "operator": [ + { + "name": "=", + "label": "=", + "function": "expr = value" + }, + { + "name": "!=", + "label": "!=", + "function": "expr != value" + }, + { + "name": ">", + "label": ">", + "function": "expr > value" + }, + { + "name": "<", + "label": "<", + "function": "expr < value" + }, + { + "name": ">=", + "label": ">=", + "function": "expr >= value" + }, + { + "name": "<=", + "label": "<=", + "function": "expr <= value" + }, + { + "name": "has", + "label": "HAS", + "function": "has(expr, value)" + }, + { + "name": "in", + "label": "IN", + "function": "expr in (values)" + }, + { + "name": "not in", + "label": "NOT IN", + "function": "expr not in (values)" + }, + { + "name": "like", + "label": "LIKE", + "function": "expr like value" + }, + { + "name": "not like", + "label": "NOT LIKE", + "function": "expr not like value" + }, + { + "name": "notEmpty", + "label": "NOT EMPTY", + "function": "notEmpty(expr)" + }, + { + "name": "empty", + "label": "EMPTY", + "function": "empty(expr)" + } + ] + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_l4_protocol", + "common_l7_protocol", + "common_data_center", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "ssl_sni", + "ssl_ja3_hash" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_subscriber_id", + "common_sled_ip", + "common_device_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_sessions", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "ssl_sni", + "ssl_ja3_hash" + ], + "filters": [ + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_l4_protocol", + "common_l7_protocol", + "common_stream_dir", + "common_direction", + "common_data_center", + "common_sled_ip", + "common_device_id", + "common_schema_type", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "ssl_sni", + "ssl_ja3_hash" + ], + "references": { + "aggregation": [ + { + "type": "int", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "long", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "float", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "double", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "string", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "date", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + } + ], + "operator": [ + { + "type": "int", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "long", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "float", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "double", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "string", + "functions": "=,!=,in,not in,like,not like,notEmpty,empty" + }, + { + "type": "date", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "timestamp", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "array", + "functions": "has" + } + ] + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "BASE": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_service_category", + "common_l7_protocol", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port" + ] + }, + "HTTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_service_category", + "common_l7_protocol", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "http_url", + "http_host", + "http_domain", + "http_request_line", + "http_response_line", + "http_request_header", + "http_response_header", + "http_request_content", + "http_response_content", + "http_request_body", + "http_response_body", + "http_request_body_key", + "http_response_body_key", + "http_proxy_flag", + "http_sequence", + "http_snapshot", + "http_cookie", + "http_referer", + "http_user_agent", + "http_content_length", + "http_content_type", + "http_set_cookie", + "http_version", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "http_url", + "common_server_port" + ] + }, + "MAIL": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "mail_protocol_type", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_subject_charset", + "mail_content", + "mail_content_charset", + "mail_attachment_name", + "mail_attachment_name_charset", + "mail_attachment_content", + "mail_eml_file", + "mail_snapshot" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "mail_from", + "mail_to", + "mail_subject" + ] + }, + "DNS": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_tc", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qdcount", + "dns_ancount", + "dns_nscount", + "dns_arcount", + "dns_qname", + "dns_qtype", + "dns_qclass", + "dns_cname", + "dns_sub", + "dns_rr" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_client_ip", + "dns_qr", + "dns_qname", + "dns_qtype" + ] + }, + "SSL": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_pinningst", + "ssl_intercept_state", + "ssl_server_side_latency", + "ssl_client_side_latency", + "ssl_server_side_version", + "ssl_client_side_version", + "ssl_cert_verify", + "ssl_error", + "ssl_con_latency_ms", + "ssl_ja3_fingerprint", + "ssl_ja3_hash", + "ssl_cert_issuer", + "ssl_cert_subject" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "ssl_sni", + "common_server_ip", + "common_server_port" + ] + }, + "QUIC": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "quic_version", + "quic_sni", + "quic_user_agent" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "quic_sni", + "common_server_ip", + "common_server_port" + ] + }, + "FTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "ftp_account", + "ftp_url", + "ftp_content", + "ftp_link_type" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "ftp_url", + "common_server_ip", + "common_server_port" + ] + }, + "BGP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "bgp_type", + "bgp_as_num", + "bgp_route" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "bgp_type", + "bgp_as_num", + "common_server_ip", + "common_server_port" + ] + }, + "SIP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration", + "sip_bye" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "common_server_ip", + "common_server_port" + ] + }, + "RTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "rtp_pcap_path" + ] + }, + "APP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "app_extra_info" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_app_id", + "common_app_label", + "app_extra_info", + "common_server_ip", + "common_server_port" + ] + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "common_schema_type" + ], + "tunnel_type": { + "GTP": [ + { + "name": "gtp_sgw_ip", + "label": "S-GW IP", + "type": "string" + }, + { + "name": "gtp_pgw_ip", + "label": "P-GW IP", + "type": "string" + }, + { + "name": "gtp_sgw_port", + "label": "S-GW Port", + "type": "int" + }, + { + "name": "gtp_pgw_port", + "label": "P-GW Port", + "type": "int" + }, + { + "name": "gtp_uplink_teid", + "label": "Uplink TEID", + "type": "long" + }, + { + "name": "gtp_downlink_teid", + "label": "Downlink TEID", + "type": "long" + } + ], + "MPLS": [ + { + "name": "mpls_c2s_direction_label", + "label": "Multiprotocol Label(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "mpls_s2c_direction_label", + "label": "Multiprotocol Label(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "VLAN": [ + { + "name": "vlan_c2s_direction_id", + "label": "VLAN Direction(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "vlan_s2c_direction_id", + "label": "VLAN Direction(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "ETHERNET": [ + { + "name": "source_mac", + "label": "Source MAC", + "type": "string" + }, + { + "name": "destination_mac", + "label": "Destination MAC", + "type": "string" + } + ], + "MULTIPATH_ETHERNET": [ + { + "name": "c2s_source_mac", + "label": "Source MAC(c2s)", + "type": "string" + }, + { + "name": "c2s_destination_mac", + "label": "Destination MAC(c2s)", + "type": "string" + }, + { + "name": "s2c_source_mac", + "label": "Source MAC(s2c)", + "type": "string" + }, + { + "name": "s2c_destination_mac", + "label": "Destination MAC(s2c)", + "type": "string" + } + ] + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn,radius_match", + "appendTo": "common_client_asn,common_subscriber_id" + } + }, + "type": "string" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "visibility": "hidden", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "BASE", + "value": "BASE" + }, + { + "code": "MAIL", + "value": "MAIL" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "SSL", + "value": "SSL" + }, + { + "code": "FTP", + "value": "FTP" + }, + { + "code": "SIP", + "value": "SIP" + }, + { + "code": "RTP", + "value": "RTP" + }, + { + "code": "APP", + "value": "APP" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "City A", + "value": "City A" + }, + { + "code": "City B", + "value": "City B" + }, + { + "code": "City C", + "value": "City C" + }, + { + "code": "City D", + "value": "City D" + }, + { + "code": "City E", + "value": "City E" + }, + { + "code": "City F", + "value": "City F" + }, + { + "code": "City G", + "value": "City G" + }, + { + "code": "City H", + "value": "City H" + }, + { + "code": "City I", + "value": "City I" + }, + { + "code": "City J", + "value": "City J" + }, + { + "code": "City K", + "value": "City K" + }, + { + "code": "City L", + "value": "City L" + }, + { + "code": "City M", + "value": "City M" + }, + { + "code": "City N", + "value": "City N" + }, + { + "code": "City O", + "value": "City O" + }, + { + "code": "City P", + "value": "City P" + }, + { + "code": "City Q", + "value": "City Q" + }, + { + "code": "City R", + "value": "City R" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Ethernet" + }, + { + "code": "8", + "value": "PPP" + }, + { + "code": "12", + "value": "CiscoHDLC" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "type": "string", + "doc": { + "allow_query": "true" + } + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string" + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string" + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "allow_query": "true", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ], + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "http_url", + "label": "HTTP.URL", + "type": "string" + }, + { + "name": "http_host", + "label": "HTTP.Host", + "doc": { + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "http_domain", + "label": "HTTP.Domain", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_request_line", + "label": "HTTP.Request Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_line", + "label": "HTTP.Response Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_request_header", + "label": "HTTP.Request Headers", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_response_header", + "label": "HTTP.Response Headers", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_request_content", + "label": "HTTP.Request Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_response_content", + "label": "HTTP.Response Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_request_body", + "label": "HTTP.Request Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_response_body", + "label": "HTTP.Response Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_request_body_key", + "label": "HTTP.Request Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_body_key", + "label": "HTTP.Response Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_proxy_flag", + "label": "HTTP.Proxy Flag", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_sequence", + "label": "HTTP.Sequence", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_snapshot", + "label": "HTTP.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_cookie", + "label": "HTTP.Cookie", + "type": "string" + }, + { + "name": "http_referer", + "label": "HTTP.Referer", + "type": "string" + }, + { + "name": "http_user_agent", + "label": "HTTP.User Agent", + "type": "string" + }, + { + "name": "http_content_length", + "label": "HTTP.Content Length", + "type": "string" + }, + { + "name": "http_content_type", + "label": "HTTP.Content Type", + "type": "string" + }, + { + "name": "http_set_cookie", + "label": "HTTP.Set Cookie", + "type": "string" + }, + { + "name": "http_version", + "label": "HTTP.Version", + "type": "string" + }, + { + "name": "http_response_latency_ms", + "label": "HTTP.Response Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "http_session_duration_ms", + "label": "HTTP.Session Duration(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "http_action_file_size", + "label": "HTTP.Action File Size", + "type": "int" + }, + { + "name": "mail_protocol_type", + "label": "Mail.Protocol Type", + "type": "string" + }, + { + "name": "mail_account", + "label": "Mail.Account", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "mail_from_cmd", + "label": "Mail.From CMD", + "type": "string" + }, + { + "name": "mail_to_cmd", + "label": "Mail.To CMD", + "type": "string" + }, + { + "name": "mail_from", + "label": "Mail.From", + "doc": { + "allow_query": "true", + "constraints": { + "type": "email" + } + }, + "type": "string" + }, + { + "name": "mail_to", + "label": "Mail.To", + "doc": { + "allow_query": "true", + "constraints": { + "type": "email" + } + }, + "type": "string" + }, + { + "name": "mail_cc", + "label": "Mail.CC", + "type": "string" + }, + { + "name": "mail_bcc", + "label": "Mail.BCC", + "type": "string" + }, + { + "name": "mail_subject", + "label": "Mail.Subject", + "doc": { + "allow_query": "true", + "format": { + "functions": "decode_of_base64", + "param": "$.mail_subject_charset" + } + }, + "type": "string" + }, + { + "name": "mail_subject_charset", + "label": "Mail.Subject Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_content", + "label": "Mail.Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_content_charset", + "label": "Mail.Content Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_attachment_name", + "label": "Mail.Attachment", + "doc": { + "format": { + "functions": "decode_of_base64", + "param": "$.mail_attachment_name_charset" + } + }, + "type": "string" + }, + { + "name": "mail_attachment_name_charset", + "label": "Mail.Attachment Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_attachment_content", + "label": "Mail.Attachment Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_eml_file", + "label": "Mail.EML File", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "mail_snapshot", + "label": "Mail.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "dns_message_id", + "label": "DNS.Message ID", + "type": "int" + }, + { + "name": "dns_qr", + "label": "DNS.QR", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "RESPONSE" + } + ] + }, + "type": "int" + }, + { + "name": "dns_opcode", + "label": "DNS.OPCODE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "IQUERY" + }, + { + "code": "2", + "value": "STATUS" + }, + { + "code": "5", + "value": "UPDATE" + } + ] + }, + "type": "int" + }, + { + "name": "dns_aa", + "label": "DNS.AA", + "type": "int" + }, + { + "name": "dns_tc", + "label": "DNS.TC", + "type": "int" + }, + { + "name": "dns_rd", + "label": "DNS.RD", + "type": "int" + }, + { + "name": "dns_ra", + "label": "DNS.RA", + "type": "int" + }, + { + "name": "dns_rcode", + "label": "DNS.RCODE", + "type": "int" + }, + { + "name": "dns_qdcount", + "label": "DNS.QDCOUNT", + "type": "int" + }, + { + "name": "dns_ancount", + "label": "DNS.ANCOUNT", + "type": "int" + }, + { + "name": "dns_nscount", + "label": "DNS.NSCOUNT", + "type": "int" + }, + { + "name": "dns_arcount", + "label": "DNS.ARCOUNT", + "type": "int" + }, + { + "name": "dns_qname", + "label": "DNS.QNAME", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "dns_qtype", + "label": "DNS.QTYPE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "A" + }, + { + "code": "2", + "value": "NS" + }, + { + "code": "5", + "value": "CNAME" + }, + { + "code": "6", + "value": "SOA" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "12", + "value": "PTR" + }, + { + "code": "13", + "value": "HINFO" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "15", + "value": "MX" + }, + { + "code": "28", + "value": "AAAA" + } + ] + }, + "type": "int" + }, + { + "name": "dns_qclass", + "label": "DNS.QCLASS", + "type": "int" + }, + { + "name": "dns_cname", + "label": "DNS.CNAME", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "dns_sub", + "label": "DNS.SUB", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "DNS" + }, + { + "code": "2", + "value": "DNSSEC" + } + ] + }, + "type": "int" + }, + { + "name": "dns_rr", + "label": "DNS.RR", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_version", + "label": "SSL.Version", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_sni", + "label": "SSL.SNI", + "doc": { + "allow_query": "true", + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "ssl_san", + "label": "SSL.SAN", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_cn", + "label": "SSL.CN", + "type": "string" + }, + { + "name": "ssl_pinningst", + "label": "SSL.Pinning", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Not Pinning" + }, + { + "code": "1", + "value": "Pinning" + }, + { + "code": "2", + "value": "Maybe Pinning" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_intercept_state", + "label": "SSL.Intercept State", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Passthrough" + }, + { + "code": "1", + "value": "Intercept" + }, + { + "code": "2", + "value": "Shutdown" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_server_side_latency", + "label": "SSL.Server Side Latency(ms)", + "type": "int" + }, + { + "name": "ssl_client_side_latency", + "label": "SSL.Client Side Latency(ms)", + "type": "int" + }, + { + "name": "ssl_server_side_version", + "label": "SSL.Server Side Version", + "type": "string" + }, + { + "name": "ssl_client_side_version", + "label": "SSL.Client Side Version", + "type": "string" + }, + { + "name": "ssl_cert_verify", + "label": "SSL.Certificate Verify", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_error", + "label": "SSL.Error", + "type": "string" + }, + { + "name": "ssl_con_latency_ms", + "label": "SSL.Connection Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "ssl_ja3_fingerprint", + "label": "SSL.JA3", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "ssl_ja3_hash", + "label": "SSL.JA3 hash", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ssl_cert_issuer", + "label": "SSL.Issuer", + "doc": { + "constraints": { + "type": "items" + } + }, + "type": "string" + }, + { + "name": "ssl_cert_subject", + "label": "SSL.Subject", + "doc": { + "constraints": { + "type": "items" + } + }, + "type": "string" + }, + { + "name": "quic_version", + "label": "QUIC.Version", + "type": "string" + }, + { + "name": "quic_sni", + "label": "QUIC.SNI", + "doc": { + "allow_query": "true", + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "quic_user_agent", + "label": "QUIC.User Agent", + "type": "string" + }, + { + "name": "ftp_account", + "label": "FTP.Account", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ftp_url", + "label": "FTP.URL", + "type": "string" + }, + { + "name": "ftp_content", + "label": "FTP.Content", + "type": "string" + }, + { + "name": "ftp_link_type", + "label": "FTP.Link Type", + "type": "string" + }, + { + "name": "bgp_type", + "label": "BGP.Type", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "bgp_as_num", + "label": "BGP.AS Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "bgp_route", + "label": "BGP.Route", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_calling_account", + "label": "VoIP.Calling Account", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_called_account", + "label": "VoIP.Called Account", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_calling_number", + "label": "VoIP.Calling Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_called_number", + "label": "VoIP.Called Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "streaming_media_url", + "label": "Streaming.Media URL", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "streaming_media_protocol", + "label": "Streaming.Media Protocol", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "app_extra_info", + "label": "APP.Extra Info", + "type": "string" + }, + { + "name": "sip_call_id", + "label": "SIP.Call-ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "sip_originator_description", + "label": "SIP.Originator", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "sip_responder_description", + "label": "SIP.Responder", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "sip_user_agent", + "label": "SIP.User-Agent", + "type": "string" + }, + { + "name": "sip_server", + "label": "SIP.Server", + "type": "string" + }, + { + "name": "sip_originator_sdp_connect_ip", + "label": "SIP.Originator IP", + "type": "string" + }, + { + "name": "sip_originator_sdp_media_port", + "label": "SIP.Originator Port", + "type": "int" + }, + { + "name": "sip_originator_sdp_media_type", + "label": "SIP.Originator Media Type", + "type": "string" + }, + { + "name": "sip_originator_sdp_content", + "label": "SIP.Originator Content", + "type": "string" + }, + { + "name": "sip_responder_sdp_connect_ip", + "label": "SIP.Responder IP", + "type": "string" + }, + { + "name": "sip_responder_sdp_media_port", + "label": "SIP.Responder Port", + "type": "int" + }, + { + "name": "sip_responder_sdp_media_type", + "label": "SIP.Responder Media Type", + "type": "string" + }, + { + "name": "sip_responder_sdp_content", + "label": "SIP.Responder Content", + "type": "string" + }, + { + "name": "sip_duration", + "label": "SIP.Duration", + "type": "int" + }, + { + "name": "sip_bye", + "label": "SIP.Bye", + "type": "string" + }, + { + "name": "rtp_payload_type_c2s", + "label": "RTP.Payload Type(c2s)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_payload_type_s2c", + "label": "RTP.Payload Type(s2c)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_pcap_path", + "label": "RTP.PCAP", + "doc": { + "constraints": { + "type": "files" + } + }, + "type": "string" + }, + { + "name": "rtp_originator_dir", + "label": "RTP.Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "unknown" + }, + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + } + ], + "visibility": "hidden" + }, + "type": "int" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_common_client_ip.json b/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_common_client_ip.json new file mode 100644 index 0000000..e407d26 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_common_client_ip.json @@ -0,0 +1,59 @@ +{ + "type": "record", + "name": "connection_record_log_common_client_ip", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "index_key": "common_client_ip" + }, + "fields": [ + { + "name": "common_log_id", + "type": "long" + }, + { + "name": "common_recv_time", + "type": "long" + }, + { + "name": "common_policy_id", + "type": "long" + }, + { + "name": "common_action", + "type": "int" + }, + { + "name": "common_server_ip", + "type": "string" + }, + { + "name": "common_client_ip", + "type": "string" + }, + { + "name": "common_sled_ip", + "type": "string" + }, + { + "name": "common_entrance_id", + "type": "int" + }, + { + "name": "common_subscriber_id", + "type": "string" + }, + { + "name": "common_stream_trace_id", + "type": "long" + }, + { + "name": "http_domain", + "type": "string" + }, + { + "name": "ssl_sni", + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_common_server_ip.json b/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_common_server_ip.json new file mode 100644 index 0000000..a63ddd8 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_common_server_ip.json @@ -0,0 +1,59 @@ +{ + "type": "record", + "name": "connection_record_log_common_server_ip", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "index_key": "common_server_ip" + }, + "fields": [ + { + "name": "common_log_id", + "type": "long" + }, + { + "name": "common_recv_time", + "type": "long" + }, + { + "name": "common_policy_id", + "type": "long" + }, + { + "name": "common_action", + "type": "int" + }, + { + "name": "common_server_ip", + "type": "string" + }, + { + "name": "common_client_ip", + "type": "string" + }, + { + "name": "common_sled_ip", + "type": "string" + }, + { + "name": "common_entrance_id", + "type": "int" + }, + { + "name": "common_subscriber_id", + "type": "string" + }, + { + "name": "common_stream_trace_id", + "type": "long" + }, + { + "name": "http_domain", + "type": "string" + }, + { + "name": "ssl_sni", + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_common_subscriber_id.json b/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_common_subscriber_id.json new file mode 100644 index 0000000..736f3f2 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_common_subscriber_id.json @@ -0,0 +1,59 @@ +{ + "type": "record", + "name": "connection_record_log_common_subscriber_id", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "index_key": "common_subscriber_id" + }, + "fields": [ + { + "name": "common_log_id", + "type": "long" + }, + { + "name": "common_recv_time", + "type": "long" + }, + { + "name": "common_policy_id", + "type": "long" + }, + { + "name": "common_action", + "type": "int" + }, + { + "name": "common_server_ip", + "type": "string" + }, + { + "name": "common_client_ip", + "type": "string" + }, + { + "name": "common_sled_ip", + "type": "string" + }, + { + "name": "common_entrance_id", + "type": "int" + }, + { + "name": "common_subscriber_id", + "type": "string" + }, + { + "name": "common_stream_trace_id", + "type": "long" + }, + { + "name": "http_domain", + "type": "string" + }, + { + "name": "ssl_sni", + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_http_domain.json b/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_http_domain.json new file mode 100644 index 0000000..2e25585 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/connection_record_log_http_domain.json @@ -0,0 +1,59 @@ +{ + "type": "record", + "name": "connection_record_log_http_domain", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "index_key": "http_domain" + }, + "fields": [ + { + "name": "common_log_id", + "type": "long" + }, + { + "name": "common_recv_time", + "type": "long" + }, + { + "name": "common_policy_id", + "type": "long" + }, + { + "name": "common_action", + "type": "int" + }, + { + "name": "common_server_ip", + "type": "string" + }, + { + "name": "common_client_ip", + "type": "string" + }, + { + "name": "common_sled_ip", + "type": "string" + }, + { + "name": "common_entrance_id", + "type": "int" + }, + { + "name": "common_subscriber_id", + "type": "string" + }, + { + "name": "common_stream_trace_id", + "type": "long" + }, + { + "name": "http_domain", + "type": "string" + }, + { + "name": "ssl_sni", + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/disks_cluster.json b/parcels/roles/services/galaxy-qgw-service/files/config/disks_cluster.json new file mode 100644 index 0000000..70777c6 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/disks_cluster.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "disks_cluster", + "fields": [ + { + "name": "name", + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/druid-filter.json b/parcels/roles/services/galaxy-qgw-service/files/config/druid-filter.json new file mode 100644 index 0000000..5c437b9 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/druid-filter.json @@ -0,0 +1,21 @@ +{ + "version": "1.0", + "name": "connection_record_log", + "namespace": "druid", + "filters": [ + { + "name":"@start", + "value": "'2021-01-11 10:00:00'" + }, + { + "name":"@end", + "value": "'2021-01-13 11:00:00'" + }, + { + "name":"@common_filter", + "value": [ + "common_client_ip='192.168.44.21'and common_server_port=443" + ] + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/druid-queries-template.sql b/parcels/roles/services/galaxy-qgw-service/files/config/druid-queries-template.sql new file mode 100644 index 0000000..0191e5c --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/druid-queries-template.sql @@ -0,0 +1,112 @@ +--Q01.All Security Event Hits +select policy_id, sum(hits) as hits from security_event_hits_log where __time >@start and __time <@end group by policy_id +--Q02.Security Event Hits with Policy ID 0 +select policy_id, sum(hits) as hits from security_event_hits_log where __time >@start and __time <@end and policy_id in (0) group by policy_id +--Q03.All Security Event Hits Trend by 5min A +select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sum(hits) as hits from security_event_hits_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') limit 10000 +--Q04.All Security Event Hits Trend by 5min B +select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as start_time, sum(hits) as hits from security_event_hits_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') limit 10000 +--Q05.Security Event Hit Time（first and last time) A +select policy_id,TIME_FORMAT(min(__time) ,'yyyy-MM-dd HH:mm:ss') as first_used, TIME_FORMAT(max(__time) ,'yyyy-MM-dd HH:mm:ss') as last_used from security_event_hits_log where policy_id in (0) group by policy_id +--Q06.Security Event Hit Time（first and last time) B +select policy_id, DATE_FORMAT(min(__time) ,'%Y-%m-%d %H:%i:%s') as first_used, DATE_FORMAT(max(__time) ,'%Y-%m-%d %H:%i:%s') as last_used from security_event_hits_log where policy_id in (0) group by policy_id +--Q07.Top 200 Security Policies +select policy_id, sum(hits) as hits from security_event_hits_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by policy_id order by hits desc limit 200 +--Q08.Top 200 Security Policies with Action +select policy_id, action, sum(hits) as hits from security_event_hits_log where __time >=@start and __time <@end group by policy_id, action order by hits desc limit 200 +--Q09.All Proxy Event Hits +select policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end group by policy_id +--Q10.Proxy Event Hits with Policy ID 0 +select policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end and policy_id=0 group by policy_id +--Q11.All Proxy Event Hits Trend by 5min A +select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sum(hits) as hits from proxy_event_hits_log where __time >= TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') limit 10000 +--Q12.All Proxy Event Hits Trend by 5min B +select FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300) as start_time, sum(hits) as hits from proxy_event_hits_log where __time >= @start and __time < @end group by FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300) limit 10000 +--Q13.Proxy Event Hit Time（first and last time) A +select policy_id,TIME_FORMAT(min(__time) ,'yyyy-MM-dd HH:mm:ss') as first_used, TIME_FORMAT(max(__time) ,'yyyy-MM-dd HH:mm:ss') as last_used from proxy_event_hits_log where policy_id in (0) group by policy_id +--Q14.Proxy Event Hit Time（first and last time) B +select policy_id, DATE_FORMAT(min(__time) ,'%Y-%m-%d %H:%i:%s') as first_used, DATE_FORMAT(max(__time) ,'%Y-%m-%d %H:%i:%s') as last_used from proxy_event_hits_log where policy_id in (0) group by policy_id +--Q15.Top 200 Proxy Policies +select policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by policy_id order by hits desc limit 200 +--Q16.Top 200 Proxy Policies with sub Action +select policy_id, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >=@start and __time <@end group by policy_id, sub_action order by hits desc limit 200 +--Q17.Proxy Action Hits +select sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by sub_action +--Q18.Proxy Action Hits Trend by 5min +select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') , sub_action limit 10000 +--Q19.Traffic Metrics Pinning Hits +SELECT sum(not_pinning_num) AS sessions, 'notPinningNum' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end UNION ALL SELECT sum(pinning_num) AS sessions, 'pinningNum' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end UNION ALL SELECT sum(maybe_pinning_num) AS sessions, 'maybePinningNum' AS type FROM traffic_metrics_log WHERE __time >= @start AND __time < @end +--Q20.Traffic Metrics Pinning Trend by 5Min +SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(pinning_num) AS sessions FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 10000 +--Q21.Traffic Metrics Not Pinning Trend by 5Min +SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(not_pinning_num) AS sessions FROM traffic_metrics_log WHERE __time>= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 10000 +--Q22.Traffic Metrics Maybe Pinning Trend by 5Min +SELECT TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') AS statisticTime, sum(maybe_pinning_num) AS sessions FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY TIME_FORMAT( MILLIS_TO_TIMESTAMP( 1000 * (TIMESTAMP_TO_MILLIS(time_floor(0.001 * TIMESTAMP_TO_MILLIS( __time) * 1000,'PT300S'))/1000)),'YYYY-MM-dd HH:mm:ss') LIMIT 10000 +--Q23.Traffic Metrics Throughput Bytes IN/OUT +select sum(total_in_bytes) as traffic_in_bytes, sum(total_out_bytes) as traffic_out_bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end +--Q24. Traffic Metrics Throughput Packets IN/OUT +select sum(total_in_packets) as traffic_in_packets, sum(total_out_packets) as traffic_out_packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end +--Q25.Traffic Metrics New Sessions +select sum(new_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end +--Q26.Traffic Metrics Bandwidth Bytes IN/OUT +select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_in_bytes' as type, sum(total_in_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_out_bytes' as type, sum(total_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') +--Q27.Traffic Metrics Bandwidth Packets IN/OUT +select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_in_packets' as type, sum(total_in_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_out_packets' as type, sum(total_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') +--Q28.Traffic Metrics New Sessions Trend by 5Min +select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'new_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') +--Q29.Traffic Metrics New and Live Sessions +select sum(new_conn_num) as new_conn_num, sum(established_conn_num) as established_conn_num from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end +--Q30.Traffic Metrics New and Live Sessions Trend by 5Min +select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'new_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'established_conn_num' as type, sum(established_conn_num) as sessions from traffic_metrics_log where __time >= TIMESTAMP @start and __time < TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') +--Q31.Traffic Metrics Security Throughput Bytes +select sum(default_in_bytes+default_out_bytes) as default_bytes, sum(allow_in_bytes+allow_out_bytes) as allow_bytes, sum(deny_in_bytes+deny_out_bytes) as deny_bytes, sum(monitor_in_bytes+monitor_out_bytes) as monitor_bytes, sum(intercept_in_bytes+intercept_out_bytes) as intercept_bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time < TIMESTAMP @end +--Q32.Traffic Metrics Security Throughput Packets +select sum(default_in_packets+default_out_packets) as default_packets, sum(allow_in_packets+allow_in_packets) as allow_packets, sum(deny_in_packets+deny_out_packets) as deny_packets, sum(monitor_in_packets+monitor_out_packets) as monitor_packets, sum(intercept_in_packets+intercept_out_packets) as intercept_packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end +--Q33.Traffic Metrics Security Throughput Sessions +select sum(default_conn_num) as default_sessions, sum(allow_conn_num) as allow_sessions, sum(deny_conn_num) as deny_sessions, sum(monitor_conn_num) as monitor_sessions, sum(intercept_conn_num) as intercept_sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end +--Q34.Traffic Metrics Security Bandwidth Bytes by 5Min +select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'default_bytes' as type, sum(default_in_bytes+default_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'allow_bytes' as type, sum(allow_in_bytes+allow_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'deny_bytes' as type, sum(deny_in_bytes+deny_out_bytes) as bytes from traffic_metrics_log where __time >= TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'monitor_bytes' as type, sum(monitor_in_bytes+monitor_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'intercept_bytes' as type, sum(intercept_in_bytes+intercept_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') +--Q35.Traffic Metrics Security Bandwidth Packets by 5Min +select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'default_packets' as type, sum(default_in_packets+default_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'allow_packets' as type, sum(allow_in_packets+allow_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'deny_packets' as type, sum(deny_in_packets+deny_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'monitor_packets' as type, sum(monitor_in_packets+monitor_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'intercept_packets' as type, sum(intercept_in_packets+intercept_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') +--Q36.Traffic Metrics Security Sessions Trend by 5Min +select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'default_conn_num' as type, sum(default_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'allow_conn_num' as type, sum(allow_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'deny_conn_num' as type, sum(deny_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'monitor_conn_num' as type, sum(monitor_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') union all select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'intercept_conn_num' as type, sum(intercept_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') +--Q37.Top 100 Client IP by Sessions +select source as client_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_client_ip_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by='sessions' group by source order by sessions desc limit 100 +--Q38.Top 100 Server IP by Sessions +select destination as server_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_server_ip_log where __time >= @start and __time < @end and order_by='sessions' group by destination order by sessions desc limit 100 +--Q39.Top 100 Internal IP by Sessions +select source as internal_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_internal_host_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by='sessions' group by source order by sessions desc limit 100 +--Q40.Top 100 External IP by Sessions +select destination as external_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_external_host_log where __time >= @start and __time < @end and order_by='sessions' group by destination order by sessions desc limit 100 +--Q41.Top 100 Domain by Bytes +select domain, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_website_domain_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by='bytes' group by domain order by bytes desc limit 100 +--Q42.Top 100 Subscriber ID by Sessions +select subscriber_id, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_user_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end and order_by='sessions' group by subscriber_id order by sessions desc limit 100 +--Q43.Top 100 Hit URLS by hits +select url,sum(session_num) as hits from top_urls_log where __time >=TIMESTAMP @start and __time <TIMESTAMP @end group by url order by hits desc limit 100 +--Q44.Proxy Event Unique ISP +SELECT policy_id, APPROX_COUNT_DISTINCT_DS_HLL(isp) as num FROM proxy_event_hits_log where __time >= @start and __time < @end group by policy_id +--Q45.Traffic Composition Metrics +SELECT APPROX_COUNT_DISTINCT_DS_HLL(ip_object) AS uniq_client_ip, SUM(one_sided_connections) AS one_sided_connections, SUM(uncategorized_bytes) AS total_uncategorized_bytes, SUM(fragmentation_packets) AS fragmentation_packets, SUM(sequence_gap_loss) AS sequence_gap_loss_bytes, SUM(s2c_byte_num+c2s_byte_num) AS summaryTotalBytes, SUM(s2c_pkt_num+c2s_pkt_num) AS summaryTotalPackets, SUM(sessions) AS summarySessions FROM traffic_summary_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end LIMIT 1 +--Q46.Traffic Composition Throughput +(SELECT SUM(c2s_byte_num + s2c_byte_num) as total_bytes, SUM(sessions) as total_sessions, (SUM(c2s_byte_num + s2c_byte_num) * 8)/((TIMESTAMP_TO_MILLIS(TIMESTAMP @end )-TIMESTAMP_TO_MILLIS(TIMESTAMP @start ))/1000) AS data_rate FROM traffic_protocol_stat_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end AND protocol_id = 'ETHERNET' LIMIT 1) UNION ALL ( SELECT SUM(sessions), 0, 0 FROM traffic_protocol_stat_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end AND protocol_id = 'ETHERNET' GROUP BY __time ORDER BY __time DESC LIMIT 1 ) +--Q47.Traffic Composition Protocol Tree +SELECT protocol_id, SUM(sessions) as sessions,SUM(c2s_byte_num) as c2s_byte_num, SUM(c2s_pkt_num) as c2s_pkt_num, SUM(s2c_byte_num) as s2c_byte_num, SUM(s2c_pkt_num) as s2c_pkt_num FROM traffic_protocol_stat_log WHERE __time >= TIMESTAMP @start AND __time < TIMESTAMP @end GROUP BY protocol_id +--Q48.Traffic Composition Protocol Tree Trend +(SELECT TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, 'PT30S', 'zero')), 'yyyy-MM-dd HH:mm:ss') as stat_time, protocol_id as type, sum(c2s_byte_num + s2c_byte_num) as bytes from traffic_protocol_stat_log where __time >= TIMESTAMP @start AND __time < TIMESTAMP @end and protocol_id = 'ETHERNET' group by TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, 'PT30S', 'zero')), 'yyyy-MM-dd HH:mm:ss'), protocol_id order by stat_time asc) union all (SELECT TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, 'PT30S', 'zero')), 'yyyy-MM-dd HH:mm:ss') as stat_time, protocol_id as type, sum(c2s_byte_num + s2c_byte_num) as bytes from traffic_protocol_stat_log where __time >= TIMESTAMP @start AND __time < TIMESTAMP @end and protocol_id like CONCAT('ETHERNET','.%') and LENGTH(protocol_id) = LENGTH(REPLACE(protocol_id,'.','')) + 1 + 0 group by TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, 'PT30S', 'zero')), 'yyyy-MM-dd HH:mm:ss'), protocol_id order by stat_time asc) +--Q49.System Quota +SELECT log_type, SUM(used_size) as used_size, SUM(max_size) * 7/10 as max_size, TIME_FORMAT(LATEST(last_storage) * 1000,'YYYY-MM-dd') as first_storage FROM ( SELECT log_type, LATEST(used_size) as used_size, LATEST(max_size) as max_size, LATEST(last_storage) as last_storage FROM sys_storage_log WHERE __time >= CURRENT_TIMESTAMP - INTERVAL '1' HOUR AND data_center != '' GROUP BY data_center,log_type ) GROUP BY log_type +--Q50.System Quota Daily Trend +select TIME_FORMAT(__time,'YYYY-MM-dd') as stat_time,log_type as type, sum(aggregate_size) as used_size from sys_storage_log where __time >= @start and __time < @end group by TIME_FORMAT(__time,'YYYY-MM-dd'), log_type +--Q51.Traffic Metrics Security Action Hits Trend +select FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1800S','zero')) as statisticTime, sum(default_in_bytes + default_out_bytes) as default_bytes, sum(default_in_packets + default_out_packets) as default_packets, sum(default_conn_num) as default_sessions, sum(allow_in_bytes + allow_out_bytes) as allow_bytes, sum(allow_in_packets + allow_out_packets) as allow_packets, sum(allow_conn_num) as allow_sessions, sum(deny_in_bytes + deny_out_bytes) as deny_bytes, sum(deny_in_packets + deny_out_packets) as deny_packets, sum(deny_conn_num) as deny_sessions, sum(monitor_in_bytes + monitor_out_bytes) as monitor_bytes, sum(monitor_in_packets + monitor_out_packets) as monitor_packets, sum(monitor_conn_num) as monitor_sessions, sum(intercept_in_bytes + intercept_out_bytes) as intercept_bytes, sum(intercept_in_packets + intercept_out_packets) as intercept_packets, sum(intercept_conn_num) as intercept_sessions from traffic_metrics_log where __time >= @start and __time < @end group by FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1800S','zero')) limit 100000 +--Q52.Traffic Metrics Proxy Action Hits Trend +SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1800S','zero')) AS statisticTime,SUM(intcp_allow_num) AS intercept_allow_conn_num,SUM(intcp_mon_num) AS intercept_monitor_conn_num,SUM(intcp_deny_num) AS intercept_deny_conn_num,SUM(intcp_rdirt_num) AS intercept_redirect_conn_num,SUM(intcp_repl_num) AS intercept_replace_conn_num,SUM(intcp_hijk_num) AS intercept_hijack_conn_num,SUM(intcp_ins_num) AS intercept_insert_conn_num FROM traffic_metrics_log WHERE __time >= @start AND __time < @end GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), 'PT1800S', 'zero')) LIMIT 100000 +--Q53.Traffic Statistics(Metrics01) +select sum(total_hit_sessions) as total_hit_sessions, sum(total_bytes_transferred) as total_bytes_transferred, sum(total_packets_transferred) as total_packets_transferred, sum(total_new_sessions) as total_new_sessions , sum(total_close_sessions) as total_close_sessions, sum(average_new_sessions_per_second) as average_new_sessions_per_second , sum(average_bytes_per_second) as average_bytes_per_second , sum(average_packets_per_second) as average_packets_per_second , COUNT(DISTINCT(device_id)) as device_num, sum(live_sessions) as average_live_sessions from ( select device_id, sum(intercept_conn_num + monitor_conn_num + deny_conn_num + allow_conn_num) as total_hit_sessions, sum(total_in_bytes + total_out_bytes) as total_bytes_transferred, sum(total_in_packets + total_out_packets) as total_packets_transferred, sum(new_conn_num) as total_new_sessions, sum(close_conn_num) as total_close_sessions, avg(nullif(new_conn_num, 0))/ 5 as average_new_sessions_per_second, avg(nullif(total_in_bytes + total_out_bytes, 0))* 8 / 5 as average_bytes_per_second, avg(nullif(total_in_packets + total_out_packets, 0))/ 5 as average_packets_per_second, avg(nullif(established_conn_num, 0)) as live_sessions from traffic_metrics_log where __time >= @start and __time < @end group by device_id) +--Q54.Traffic Statistics(Metrics02) +select FROM_UNIXTIME(stat_time) as max_active_date_by_sessions, total_live_sessions as max_live_sessions from ( select stat_time, sum(live_sessions) as total_live_sessions from ( select TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), 'P1D') as stat_time, device_id, avg(established_conn_num) as live_sessions from traffic_metrics_log where __time >= @start and __time<@end group by TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), 'P1D'), device_id) group by stat_time order by total_live_sessions desc limit 1 ) +--Q55.Traffic Summary(Bandwidth Trend) +select * from ( select DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s') as stat_time,'traffic_in_bytes' as type, sum(total_in_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s'), 'traffic_in_bytes' union all select DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s') as stat_time,'traffic_out_bytes' as type,sum(total_out_bytes) as bytes from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s'),'traffic_out_bytes' ) order by stat_time asc limit 100000 +--Q56.Traffic Summary(Sessions Trend) +select DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s') as stat_time, 'total_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= @start and __time < @end group by DATE_FORMAT(FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1h','zero')),'%Y-%m-%d %H:%i:%s'), 'total_conn_num' order by stat_time asc limit 10000
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/engine-filter.json b/parcels/roles/services/galaxy-qgw-service/files/config/engine-filter.json new file mode 100644 index 0000000..0fda6d4 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/engine-filter.json @@ -0,0 +1,53 @@ +{ + "version": "1.0", + "name": "connection_record_log", + "namespace": "Engine", + "filters": [ + { + "name":"@start", + "value": "'2021-01-11 10:00:00'" + }, + { + "name":"@end", + "value": "'2021-01-13 11:00:00'" + }, + { + "name":"@common_filter", + "value": [ + "common_log_id=1153021139190754263", + "common_client_ip='36.189.226.21'", + "common_internal_ip='223.116.37.192'", + "common_server_ip='8.8.8.8'", + "common_external_ip='111.10.53.14'", + "common_client_port=52607", + "common_server_port=443", + "common_c2s_pkt_num>5", + "common_s2c_pkt_num>5", + "common_c2s_byte_num>100", + "common_s2c_byte_num<200", + "common_schema_type='DNS'", + "common_establish_latency_ms>200", + "common_con_duration_ms>10000", + "common_stream_trace_id=1153021139190754263", + "common_tcp_client_isn=2857077935", + "common_tcp_server_isn=0", + "http_domain='microsoft.com'", + "mail_account='[email protected]'", + "mail_subject='test'", + "dns_qname='qbwup.imtt.qq.com'", + "ssl_sni='note.youdao.com'", + "ssl_con_latency_ms>100", + "ssl_ja3_hash='a0e9f5d64349fb13191bc781f81f42e1'", + "common_client_ip='36.189.226.21' and common_server_ip='8.8.8.8'", + "common_server_ip='111.10.53.14' and common_server_port=443", + "mail_account like 'abc@%'", + "http_domain like '%baidu.com%'", + "ssl_sni like '%youdao.com'", + "common_client_ip in ('36.189.226.21','111.10.53.14')", + "common_server_port not in (80,443)", + "notEmpty(http_domain)", + "http_domain not like '%microsoft.com'" + ] + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/engine-queries-template.sql b/parcels/roles/services/galaxy-qgw-service/files/config/engine-queries-template.sql new file mode 100644 index 0000000..b560be9 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/engine-queries-template.sql @@ -0,0 +1,68 @@ +--Q01.CK DateTime +select toDateTime(common_recv_time) as common_recv_time from connection_record_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) limit 20 +--Q02.Standard DateTime +select FROM_UNIXTIME(common_recv_time) as common_recv_time from connection_record_log where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) limit 20 +--Q03.count(1) +select count(1) from connection_record_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) +--Q04.count(*) +select count(*) from connection_record_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) +--Q05.UDF APPROX_COUNT_DISTINCT_DS_HLL +SELECT policy_id, APPROX_COUNT_DISTINCT_DS_HLL(isp) as num FROM proxy_event_hits_log where __time >= @start and __time < @end and policy_id=0 group by policy_id +--Q06.UDF TIME_FLOOR_WITH_FILL +select TIME_FLOOR_WITH_FILL(common_recv_time,'PT5M','previous') as stat_time from connection_record_log where common_recv_time > @start and common_recv_time < @end group by stat_time +--Q07.UDF GEO IP +select IP_TO_GEO(common_client_ip) as geo,IP_TO_CITY(common_server_ip) as city,IP_TO_COUNTRY(common_server_ip) as country from connection_record_log limit 10 +--Q08.Special characters +select * from connection_record_log where (common_protocol_label ='/$' or common_client_ip like'%') limit 10 +--Q09.Federation Query +select * from (select FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(common_recv_time,'PT5M','zero')) as stat_time from connection_record_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) group by stat_time order by stat_time asc) +--Q10.Catalog Database +select * from tsg_galaxy_v3.connection_record_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) limit 20 +--Q11.Session Record Logs +select * from connection_record_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) AND @common_filter order by common_recv_time desc limit 20 +--Q12.Live Session Record Logs +select * from live_session_record_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) AND @common_filter order by common_recv_time desc limit 20 +--Q13.Transaction Record Logs +select * from transaction_record_log where common_recv_time >= toDateTime(@start) and common_recv_time< toDateTime(@end) order by common_recv_time desc limit 20 +--Q14.Security Event Logs +select * from security_event_log where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) AND @common_filter order by common_recv_time desc limit 0,20 +--Q15.Proxy Event Logs +select * from proxy_event_log where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) order by common_recv_time desc limit 0,20 +--Q16.Radius Record Logs +select * from radius_record_log where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) order by common_recv_time desc limit 0,20 +--Q17.GTPC Record Logs +select * from gtpc_record_log where common_recv_time >= UNIX_TIMESTAMP(@start) and common_recv_time< UNIX_TIMESTAMP(@end) order by common_recv_time desc limit 0,20 +--Q18.Security Event Logs with fields +select FROM_UNIXTIME(common_recv_time) as common_recv_time,common_log_id,common_policy_id,common_subscriber_id,common_client_ip,common_client_port,common_l4_protocol,common_address_type,common_server_ip,common_server_port,common_action,common_direction,common_sled_ip,common_client_location,common_client_asn,common_server_location,common_server_asn,common_c2s_pkt_num,common_s2c_pkt_num,common_c2s_byte_num,common_s2c_byte_num,common_schema_type,common_sub_action,common_device_id, FROM_UNIXTIME(common_start_time) as common_start_time, FROM_UNIXTIME(common_end_time) as common_end_time,common_establish_latency_ms,common_con_duration_ms,common_stream_dir,common_stream_trace_id,http_url,http_host,http_domain,http_request_body,http_response_body,http_cookie,http_referer,http_user_agent,http_content_length,http_content_type,http_set_cookie,http_version,http_response_latency_ms,http_action_file_size,http_session_duration_ms,mail_protocol_type,mail_account,mail_from_cmd,mail_to_cmd,mail_from,mail_to,mail_cc,mail_bcc,mail_subject,mail_attachment_name,mail_eml_file,dns_message_id,dns_qr,dns_opcode,dns_aa,dns_tc,dns_rd,dns_ra,dns_rcode,dns_qdcount,dns_ancount,dns_nscount,dns_arcount,dns_qname,dns_qtype,dns_qclass,dns_cname,dns_sub,dns_rr,ssl_sni,ssl_san,ssl_cn,ssl_pinningst,ssl_intercept_state,ssl_server_side_latency,ssl_client_side_latency,ssl_server_side_version,ssl_client_side_version,ssl_cert_verify,ssl_error,quic_version,quic_sni,quic_user_agent,ftp_account,ftp_url,ftp_content from security_event_log where common_recv_time >= @start and common_recv_time < @end order by common_recv_time desc limit 10000 +--Q19.Radius ON/OFF Logs For Frame IP +select framed_ip, arraySlice(groupUniqArray(concat(toString(event_timestamp),':', if(acct_status_type=1,'start','stop'))),1,100000) as timeseries from radius_onff_log where event_timestamp >=toDateTime(@start) and event_timestamp <toDateTime(@end) group by framed_ip limit 20 +--Q20.Radius ON/OFF Logs For Account +select account, arraySlice(groupUniqArray(concat(toString(event_timestamp),':', if(acct_status_type=1,'start','stop'))),1,100000) as timeseries from radius_onff_log where event_timestamp >= @start and event_timestamp < @end group by account +--Q21.Radius ON/OFF Logs total Account number +select count(distinct(framed_ip)) as active_ip_num , sum(acct_session_time) as online_duration from (select any(framed_ip) as framed_ip ,max(acct_session_time) as acct_session_time from radius_onff_log where account='000jS' and event_timestamp >= @start and event_timestamp < @end group by acct_session_id) +--Q22.Radius ON/OFF Logs Account Access Detail +select max(if(acct_status_type=1,event_timestamp,0)) as start_time,max(if(acct_status_type=2,event_timestamp,0)) as end_time, any(framed_ip) as ip,max(acct_session_time) as online_duration from radius_onff_log where event_timestamp >= @start and event_timestamp < @end group by acct_session_id order by start_time desc limit 200 +--Q23.Report for Client IP +select common_client_ip, count(*) as sessions from connection_record_log where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@end)) group by common_client_ip order by sessions desc limit 0,100 +--Q24.Report for Server IP +select common_server_ip, count(*) as sessions from connection_record_log where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) group by common_server_ip order by sessions desc limit 0,100 +--Q25.Report for SSL SNI +select ssl_sni, count(*) as sessions from connection_record_log where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) group by ssl_sni order by sessions desc limit 0,100 +--Q26.Report for SSL APP +select common_app_label as applicaiton, count(*) as sessions from connection_record_log where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) group by applicaiton order by sessions desc limit 0,100 +--Q27.Report for Domains +select http_domain AS domain,SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM connection_record_log WHERE common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(domain) GROUP BY domain ORDER BY bytes DESC LIMIT 100 +--Q28.Report for Domains with unique Client IP +select toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300) as stat_time, http_domain, uniq (common_client_ip) as nums from connection_record_log where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and http_domain in (select http_domain from connection_record_log where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_domain) group by http_domain order by SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) desc limit 10 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300), http_domain order by stat_time asc limit 500 +--Q29. Report for HTTP Host +SELECT http_host as host, SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM connection_record_log WHERE common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(http_host) GROUP BY host ORDER BY bytes DESC limit 100 union all SELECT 'totals' as host, SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes, SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes, SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes from connection_record_log where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(http_host) +--Q30.Report for HTTP/HTTPS URLS with Sessions +SELECT http_url AS url,count(*) AS sessions FROM proxy_event_log WHERE common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_url) GROUP BY url ORDER BY sessions DESC LIMIT 100 +--Q31.Report for HTTP/HTTPS URLS with UNIQUE Client IP +select toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300) as stat_time, http_url, count(distinct(common_client_ip)) as nums from proxy_event_log where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and http_url IN (select http_url from proxy_event_log where common_recv_time >= toStartOfDay(toDateTime(@start))-86400 AND common_recv_time < toStartOfDay(toDateTime(@start)) and notEmpty(http_url) group by http_url order by count(*) desc limit 10 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300), http_url order by stat_time asc limit 500 +--Q32.Report for Subscriber ID with Sessions +select common_subscriber_id as user, count(*) as sessions from connection_record_log where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(user) group by common_subscriber_id order by sessions desc limit 0,100 +--Q33.Report for Subscriber ID with Bandwidth +SELECT common_subscriber_id as user,SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM connection_record_log WHERE common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start)) and notEmpty(user) GROUP BY user ORDER BY bytes DESC LIMIT 100 +--Q34.Report Unique Endpoints +select uniq(common_client_ip) as "Client IP",uniq(common_server_ip) as "Server IP",uniq(common_internal_ip) as "Internal IP",uniq(common_external_ip) as "External IP",uniq(http_domain) as "Domain",uniq(ssl_sni) as "SNI" from connection_record_log where common_recv_time>= toStartOfDay(toDateTime(@start))-604800 and common_recv_time< toStartOfDay(toDateTime(@start))
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/gtpc_record_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/gtpc_record_log.json new file mode 100644 index 0000000..d64604d --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/gtpc_record_log.json @@ -0,0 +1,1488 @@ +{ + "type": "record", + "name": "gtpc_record_log", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "functions": { + "aggregation": [ + { + "name": "COUNT", + "label": "COUNT", + "function": "count(expr)" + }, + { + "name": "COUNT_DISTINCT", + "label": "COUNT_DISTINCT", + "function": "count(distinct expr)" + }, + { + "name": "AVG", + "label": "AVG", + "function": "avg(expr)" + }, + { + "name": "SUM", + "label": "SUM", + "function": "sum(expr)" + }, + { + "name": "MAX", + "label": "MAX", + "function": "max(expr)" + }, + { + "name": "MIN", + "label": "MIN", + "function": "min(expr)" + } + ], + "operator": [ + { + "name": "=", + "label": "=", + "function": "expr = value" + }, + { + "name": "!=", + "label": "!=", + "function": "expr != value" + }, + { + "name": ">", + "label": ">", + "function": "expr > value" + }, + { + "name": "<", + "label": "<", + "function": "expr < value" + }, + { + "name": ">=", + "label": ">=", + "function": "expr >= value" + }, + { + "name": "<=", + "label": "<=", + "function": "expr <= value" + }, + { + "name": "has", + "label": "HAS", + "function": "has(expr, value)" + }, + { + "name": "in", + "label": "IN", + "function": "expr in (values)" + }, + { + "name": "not in", + "label": "NOT IN", + "function": "expr not in (values)" + }, + { + "name": "like", + "label": "LIKE", + "function": "expr like value" + }, + { + "name": "not like", + "label": "NOT LIKE", + "function": "expr not like value" + }, + { + "name": "notEmpty", + "label": "NOT EMPTY", + "function": "notEmpty(expr)" + }, + { + "name": "empty", + "label": "EMPTY", + "function": "empty(expr)" + } + ] + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_l4_protocol", + "common_l7_protocol", + "common_data_center", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "gtp_version", + "gtp_apn", + "gtp_imei", + "gtp_imsi", + "gtp_phone_number" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_sled_ip", + "common_device_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_sessions", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "gtp_version", + "gtp_apn", + "gtp_imei", + "gtp_imsi", + "gtp_phone_number" + ], + "filters": [ + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_l4_protocol", + "common_l7_protocol", + "common_stream_dir", + "common_direction", + "common_data_center", + "common_sled_ip", + "common_device_id", + "common_schema_type", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "gtp_version", + "gtp_apn", + "gtp_imei", + "gtp_imsi", + "gtp_phone_number", + "gtp_end_user_ipv4", + "gtp_end_user_ipv6", + "gtp_uplink_teid", + "gtp_downlink_teid", + "gtp_msg_type" + ], + "references": { + "aggregation": [ + { + "type": "int", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "long", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "float", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "double", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "string", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "date", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + } + ], + "operator": [ + { + "type": "int", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "long", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "float", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "double", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "string", + "functions": "=,!=,in,not in,like,not like,notEmpty,empty" + }, + { + "type": "date", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "timestamp", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "array", + "functions": "has" + } + ] + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "GTP-C": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_app_surrogate_id", + "common_app_surrogate_id", + "common_service_category", + "common_l7_protocol", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "gtp_version", + "gtp_apn", + "gtp_imei", + "gtp_imsi", + "gtp_phone_number", + "gtp_end_user_ipv4", + "gtp_end_user_ipv6", + "gtp_uplink_teid", + "gtp_downlink_teid", + "gtp_msg_type" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "gtp_version", + "gtp_msg_type", + "gtp_imsi", + "gtp_imei", + "gtp_phone_number", + "common_client_ip", + "common_server_ip" + ] + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "gtp_version", + "gtp_msg_type", + "gtp_imsi", + "gtp_imei", + "gtp_phone_number", + "common_client_ip", + "common_server_ip" + ], + "tunnel_type": { + "GTP": [ + { + "name": "gtp_sgw_ip", + "label": "S-GW IP", + "type": "string" + }, + { + "name": "gtp_pgw_ip", + "label": "P-GW IP", + "type": "string" + }, + { + "name": "gtp_sgw_port", + "label": "S-GW Port", + "type": "int" + }, + { + "name": "gtp_pgw_port", + "label": "P-GW Port", + "type": "int" + }, + { + "name": "gtp_uplink_teid", + "label": "Uplink TEID", + "type": "long" + }, + { + "name": "gtp_downlink_teid", + "label": "Downlink TEID", + "type": "long" + } + ], + "MPLS": [ + { + "name": "mpls_c2s_direction_label", + "label": "Multiprotocol Label(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "mpls_s2c_direction_label", + "label": "Multiprotocol Label(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "VLAN": [ + { + "name": "vlan_c2s_direction_id", + "label": "VLAN Direction(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "vlan_s2c_direction_id", + "label": "VLAN Direction(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "ETHERNET": [ + { + "name": "source_mac", + "label": "Source MAC", + "type": "string" + }, + { + "name": "destination_mac", + "label": "Destination MAC", + "type": "string" + } + ], + "MULTIPATH_ETHERNET": [ + { + "name": "c2s_source_mac", + "label": "Source MAC(c2s)", + "type": "string" + }, + { + "name": "c2s_destination_mac", + "label": "Destination MAC(c2s)", + "type": "string" + }, + { + "name": "s2c_source_mac", + "label": "Source MAC(s2c)", + "type": "string" + }, + { + "name": "s2c_destination_mac", + "label": "Destination MAC(s2c)", + "type": "string" + } + ] + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_client_asn" + } + }, + "type": "string" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "visibility": "hidden", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "GTP-C", + "value": "GTP-C" + } + ] + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "City A", + "value": "City A" + }, + { + "code": "City B", + "value": "City B" + }, + { + "code": "City C", + "value": "City C" + }, + { + "code": "City D", + "value": "City D" + }, + { + "code": "City E", + "value": "City E" + }, + { + "code": "City F", + "value": "City F" + }, + { + "code": "City G", + "value": "City G" + }, + { + "code": "City H", + "value": "City H" + }, + { + "code": "City I", + "value": "City I" + }, + { + "code": "City J", + "value": "City J" + }, + { + "code": "City K", + "value": "City K" + }, + { + "code": "City L", + "value": "City L" + }, + { + "code": "City M", + "value": "City M" + }, + { + "code": "City N", + "value": "City N" + }, + { + "code": "City O", + "value": "City O" + }, + { + "code": "City P", + "value": "City P" + }, + { + "code": "City Q", + "value": "City Q" + }, + { + "code": "City R", + "value": "City R" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Ethernet" + }, + { + "code": "8", + "value": "PPP" + }, + { + "code": "12", + "value": "CiscoHDLC" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "type": "string" + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string" + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "visibility": "disabled", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ], + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "gtp_version", + "label": "Version", + "type": "string" + }, + { + "name": "gtp_apn", + "label": "APN", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "gtp_imei", + "label": "IMEI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "gtp_imsi", + "label": "IMSI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "gtp_phone_number", + "label": "Phone Number", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "gtp_uplink_teid", + "label": "Uplink TEID", + "type": "long" + }, + { + "name": "gtp_downlink_teid", + "label": "Downlink TEID", + "type": "long" + }, + { + "name": "gtp_msg_type", + "label": "Message Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "create", + "value": "create" + }, + { + "code": "modify", + "value": "modify" + }, + { + "code": "delete", + "value": "delete" + } + ] + }, + "type": "string" + }, + { + "name": "gtp_end_user_ipv4", + "label": "End User Address V4", + "type": "string" + }, + { + "name": "gtp_end_user_ipv6", + "label": "End User Address V6", + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/liveChart_history.json b/parcels/roles/services/galaxy-qgw-service/files/config/liveChart_history.json new file mode 100644 index 0000000..a1a8403 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/liveChart_history.json @@ -0,0 +1,152 @@ +{ + "type": "record", + "name": "liveChart_history", + "in": "CONNECTION-RECORD-LOG", + "out": "TRAFFIC-PROTOCOL-STAT-LOG", + "task": "Protocol-Distribution", + "doc": { + "timestamp": { + "name": "stat_time", + "type": "long" + }, + "dimensions": [ + { + "name": "protocol_id", + "fieldName": "common_protocol_label", + "type": "string" + }, + { + "name": "entrance_id", + "fieldName": "common_entrance_id", + "type": "string" + }, + { + "name": "isp", + "fieldName": "common_isp", + "type": "string" + }, + { + "name": "data_center", + "fieldName": "common_data_center", + "type": "string" + } + ], + "metrics": [ + { + "function": "sum", + "name": "sessions", + "fieldName": "common_sessions", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_byte_num", + "fieldName": "common_c2s_byte_diff", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_byte_num", + "fieldName": "common_s2c_byte_diff", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_pkt_num", + "fieldName": "common_c2s_pkt_diff", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_pkt_num", + "fieldName": "common_s2c_pkt_diff", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_ipfrag_num", + "fieldName": "common_c2s_ipfrag_num", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_ipfrag_num", + "fieldName": "common_s2c_ipfrag_num", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_tcp_lostlen", + "fieldName": "common_c2s_tcp_lostlen", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_tcp_lostlen", + "fieldName": "common_s2c_tcp_lostlen", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_tcp_unorder_num", + "fieldName": "common_c2s_tcp_unorder_num", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_tcp_unorder_num", + "fieldName": "common_s2c_tcp_unorder_num", + "type": "long" + }, + { + "function": "disCount", + "name": "unique_sip_num", + "fieldName": "common_server_ip", + "type": "long" + }, + { + "function": "disCount", + "name": "unique_cip_num", + "fieldName": "common_client_ip", + "type": "long" + } + ], + "filters": [ + { + "fieldName": "common_protocol_label", + "type": "notempty" + } + ], + "transforms": [ + { + "function": "combination", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "common_l7_protocol,." + }, + { + "function": "combination", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "common_app_label,." + }, + { + "function": "hierarchy", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "." + } + ], + "action": [ + { + "label": "Default", + "metrics": "sessions,c2s_byte_num,s2c_byte_num,c2s_pkt_num,s2c_pkt_num,c2s_ipfrag_num,s2c_ipfrag_num,c2s_tcp_lostlen,s2c_tcp_lostlen,c2s_tcp_unorder_num,s2c_tcp_unorder_num" + } + ], + "granularity": { + "type": "period", + "period": "5M" + } + }, + "fields": [] +} diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/liveChart_live.json b/parcels/roles/services/galaxy-qgw-service/files/config/liveChart_live.json new file mode 100644 index 0000000..d490f3e --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/liveChart_live.json @@ -0,0 +1,152 @@ +{ + "type": "record", + "name": "liveChart_live", + "in": "LIVE-SESSION-RECORD-LOG", + "out": "TRAFFIC-PROTOCOL-STAT-LOG", + "task": "Protocol-Distribution", + "doc": { + "timestamp": { + "name": "stat_time", + "type": "long" + }, + "dimensions": [ + { + "name": "protocol_id", + "fieldName": "common_protocol_label", + "type": "string" + }, + { + "name": "entrance_id", + "fieldName": "common_entrance_id", + "type": "string" + }, + { + "name": "isp", + "fieldName": "common_isp", + "type": "string" + }, + { + "name": "data_center", + "fieldName": "common_data_center", + "type": "string" + } + ], + "metrics": [ + { + "function": "sum", + "name": "sessions", + "fieldName": "common_sessions", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_byte_num", + "fieldName": "common_c2s_byte_diff", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_byte_num", + "fieldName": "common_s2c_byte_diff", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_pkt_num", + "fieldName": "common_c2s_pkt_diff", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_pkt_num", + "fieldName": "common_s2c_pkt_diff", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_ipfrag_num", + "fieldName": "common_c2s_ipfrag_num", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_ipfrag_num", + "fieldName": "common_s2c_ipfrag_num", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_tcp_lostlen", + "fieldName": "common_c2s_tcp_lostlen", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_tcp_lostlen", + "fieldName": "common_s2c_tcp_lostlen", + "type": "long" + }, + { + "function": "sum", + "name": "c2s_tcp_unorder_num", + "fieldName": "common_c2s_tcp_unorder_num", + "type": "long" + }, + { + "function": "sum", + "name": "s2c_tcp_unorder_num", + "fieldName": "common_s2c_tcp_unorder_num", + "type": "long" + }, + { + "function": "disCount", + "name": "unique_sip_num", + "fieldName": "common_server_ip", + "type": "long" + }, + { + "function": "disCount", + "name": "unique_cip_num", + "fieldName": "common_client_ip", + "type": "long" + } + ], + "filters": [ + { + "fieldName": "common_protocol_label", + "type": "notempty" + } + ], + "transforms": [ + { + "function": "combination", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "common_l7_protocol,." + }, + { + "function": "combination", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "common_app_label,." + }, + { + "function": "hierarchy", + "name": "protocol_id", + "fieldName": "common_protocol_label", + "parameters": "." + } + ], + "action": [ + { + "label": "Default", + "metrics": "c2s_byte_num,s2c_byte_num,c2s_pkt_num,s2c_pkt_num" + } + ], + "granularity": { + "type": "period", + "period": "5M" + } + }, + "fields": [] +} diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/live_session_record_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/live_session_record_log.json new file mode 100644 index 0000000..00bf742 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/live_session_record_log.json @@ -0,0 +1,3610 @@ +{ + "type": "record", + "name": "live_session_record_log", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "functions": { + "aggregation": [ + { + "name": "COUNT", + "label": "COUNT", + "function": "count(expr)" + }, + { + "name": "COUNT_DISTINCT", + "label": "COUNT_DISTINCT", + "function": "count(distinct expr)" + }, + { + "name": "AVG", + "label": "AVG", + "function": "avg(expr)" + }, + { + "name": "SUM", + "label": "SUM", + "function": "sum(expr)" + }, + { + "name": "MAX", + "label": "MAX", + "function": "max(expr)" + }, + { + "name": "MIN", + "label": "MIN", + "function": "min(expr)" + } + ], + "operator": [ + { + "name": "=", + "label": "=", + "function": "expr = value" + }, + { + "name": "!=", + "label": "!=", + "function": "expr != value" + }, + { + "name": ">", + "label": ">", + "function": "expr > value" + }, + { + "name": "<", + "label": "<", + "function": "expr < value" + }, + { + "name": ">=", + "label": ">=", + "function": "expr >= value" + }, + { + "name": "<=", + "label": "<=", + "function": "expr <= value" + }, + { + "name": "has", + "label": "HAS", + "function": "has(expr, value)" + }, + { + "name": "in", + "label": "IN", + "function": "expr in (values)" + }, + { + "name": "not in", + "label": "NOT IN", + "function": "expr not in (values)" + }, + { + "name": "like", + "label": "LIKE", + "function": "expr like value" + }, + { + "name": "not like", + "label": "NOT LIKE", + "function": "expr not like value" + }, + { + "name": "notEmpty", + "label": "NOT EMPTY", + "function": "notEmpty(expr)" + }, + { + "name": "empty", + "label": "EMPTY", + "function": "empty(expr)" + } + ] + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_l4_protocol", + "common_l7_protocol", + "common_data_center", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url", + "ssl_sni", + "ssl_ja3_hash" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_subscriber_id", + "common_sled_ip", + "common_device_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_sessions", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url", + "ssl_sni", + "ssl_ja3_hash" + ], + "filters": [ + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_l4_protocol", + "common_l7_protocol", + "common_stream_dir", + "common_direction", + "common_data_center", + "common_sled_ip", + "common_device_id", + "common_schema_type", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url", + "ssl_sni", + "ssl_ja3_hash" + ], + "references": { + "aggregation": [ + { + "type": "int", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "long", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "float", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "double", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "string", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "date", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + } + ], + "operator": [ + { + "type": "int", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "long", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "float", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "double", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "string", + "functions": "=,!=,in,not in,like,not like,notEmpty,empty" + }, + { + "type": "date", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "timestamp", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "array", + "functions": "has" + } + ] + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "BASE": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_service_category", + "common_l7_protocol", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port" + ] + }, + "HTTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_service_category", + "common_l7_protocol", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "http_url", + "http_host", + "http_domain", + "http_request_line", + "http_response_line", + "http_request_header", + "http_response_header", + "http_request_content", + "http_response_content", + "http_request_body", + "http_response_body", + "http_request_body_key", + "http_response_body_key", + "http_proxy_flag", + "http_sequence", + "http_snapshot", + "http_cookie", + "http_referer", + "http_user_agent", + "http_content_length", + "http_content_type", + "http_set_cookie", + "http_version", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "http_url", + "common_server_port" + ] + }, + "MAIL": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "mail_protocol_type", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_subject_charset", + "mail_content", + "mail_content_charset", + "mail_attachment_name", + "mail_attachment_name_charset", + "mail_attachment_content", + "mail_eml_file", + "mail_snapshot" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "mail_from", + "mail_to", + "mail_subject" + ] + }, + "DNS": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_tc", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qdcount", + "dns_ancount", + "dns_nscount", + "dns_arcount", + "dns_qname", + "dns_qtype", + "dns_qclass", + "dns_cname", + "dns_sub", + "dns_rr" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_client_ip", + "dns_qr", + "dns_qname", + "dns_qtype" + ] + }, + "SSL": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_pinningst", + "ssl_intercept_state", + "ssl_server_side_latency", + "ssl_client_side_latency", + "ssl_server_side_version", + "ssl_client_side_version", + "ssl_cert_verify", + "ssl_error", + "ssl_con_latency_ms", + "ssl_ja3_fingerprint", + "ssl_ja3_hash", + "ssl_cert_issuer", + "ssl_cert_subject" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "ssl_sni", + "common_server_ip", + "common_server_port" + ] + }, + "QUIC": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "quic_version", + "quic_sni", + "quic_user_agent" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "quic_sni", + "common_server_ip", + "common_server_port" + ] + }, + "FTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "ftp_account", + "ftp_url", + "ftp_content", + "ftp_link_type" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "ftp_url", + "common_server_ip", + "common_server_port" + ] + }, + "BGP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "bgp_type", + "bgp_as_num", + "bgp_route" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "bgp_type", + "bgp_as_num", + "common_server_ip", + "common_server_port" + ] + }, + "SIP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration", + "sip_bye" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "common_server_ip", + "common_server_port" + ] + }, + "RTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "rtp_pcap_path", + "rtp_originator_dir" + ] + }, + "APP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "app_extra_info" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_app_id", + "common_app_label", + "app_extra_info", + "common_server_ip", + "common_server_port" + ] + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "common_schema_type" + ], + "tunnel_type": { + "GTP": [ + { + "name": "gtp_sgw_ip", + "label": "S-GW IP", + "type": "string" + }, + { + "name": "gtp_pgw_ip", + "label": "P-GW IP", + "type": "string" + }, + { + "name": "gtp_sgw_port", + "label": "S-GW Port", + "type": "int" + }, + { + "name": "gtp_pgw_port", + "label": "P-GW Port", + "type": "int" + }, + { + "name": "gtp_uplink_teid", + "label": "Uplink TEID", + "type": "long" + }, + { + "name": "gtp_downlink_teid", + "label": "Downlink TEID", + "type": "long" + } + ], + "MPLS": [ + { + "name": "mpls_c2s_direction_label", + "label": "Multiprotocol Label(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "mpls_s2c_direction_label", + "label": "Multiprotocol Label(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "VLAN": [ + { + "name": "vlan_c2s_direction_id", + "label": "VLAN Direction(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "vlan_s2c_direction_id", + "label": "VLAN Direction(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "ETHERNET": [ + { + "name": "source_mac", + "label": "Source MAC", + "type": "string" + }, + { + "name": "destination_mac", + "label": "Destination MAC", + "type": "string" + } + ], + "MULTIPATH_ETHERNET": [ + { + "name": "c2s_source_mac", + "label": "Source MAC(c2s)", + "type": "string" + }, + { + "name": "c2s_destination_mac", + "label": "Destination MAC(c2s)", + "type": "string" + }, + { + "name": "s2c_source_mac", + "label": "Source MAC(s2c)", + "type": "string" + }, + { + "name": "s2c_destination_mac", + "label": "Destination MAC(s2c)", + "type": "string" + } + ] + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn,radius_match", + "appendTo": "common_client_asn,common_subscriber_id" + } + }, + "type": "string" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "visibility": "hidden", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "BASE", + "value": "BASE" + }, + { + "code": "MAIL", + "value": "MAIL" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "SSL", + "value": "SSL" + }, + { + "code": "FTP", + "value": "FTP" + }, + { + "code": "SIP", + "value": "SIP" + }, + { + "code": "RTP", + "value": "RTP" + }, + { + "code": "APP", + "value": "APP" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "City A", + "value": "City A" + }, + { + "code": "City B", + "value": "City B" + }, + { + "code": "City C", + "value": "City C" + }, + { + "code": "City D", + "value": "City D" + }, + { + "code": "City E", + "value": "City E" + }, + { + "code": "City F", + "value": "City F" + }, + { + "code": "City G", + "value": "City G" + }, + { + "code": "City H", + "value": "City H" + }, + { + "code": "City I", + "value": "City I" + }, + { + "code": "City J", + "value": "City J" + }, + { + "code": "City K", + "value": "City K" + }, + { + "code": "City L", + "value": "City L" + }, + { + "code": "City M", + "value": "City M" + }, + { + "code": "City N", + "value": "City N" + }, + { + "code": "City O", + "value": "City O" + }, + { + "code": "City P", + "value": "City P" + }, + { + "code": "City Q", + "value": "City Q" + }, + { + "code": "City R", + "value": "City R" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Ethernet" + }, + { + "code": "8", + "value": "PPP" + }, + { + "code": "12", + "value": "CiscoHDLC" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "type": "string" + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string" + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "allow_query": "true", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ], + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "http_url", + "label": "HTTP.URL", + "type": "string" + }, + { + "name": "http_host", + "label": "HTTP.Host", + "doc": { + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "http_domain", + "label": "HTTP.Domain", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_request_line", + "label": "HTTP.Request Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_line", + "label": "HTTP.Response Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_request_header", + "label": "HTTP.Request Headers", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_response_header", + "label": "HTTP.Response Headers", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_request_content", + "label": "HTTP.Request Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_response_content", + "label": "HTTP.Response Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_request_body", + "label": "HTTP.Request Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_response_body", + "label": "HTTP.Response Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_request_body_key", + "label": "HTTP.Request Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_body_key", + "label": "HTTP.Response Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_proxy_flag", + "label": "HTTP.Proxy Flag", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_sequence", + "label": "HTTP.Sequence", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_snapshot", + "label": "HTTP.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_cookie", + "label": "HTTP.Cookie", + "type": "string" + }, + { + "name": "http_referer", + "label": "HTTP.Referer", + "type": "string" + }, + { + "name": "http_user_agent", + "label": "HTTP.User Agent", + "type": "string" + }, + { + "name": "http_content_length", + "label": "HTTP.Content Length", + "type": "string" + }, + { + "name": "http_content_type", + "label": "HTTP.Content Type", + "type": "string" + }, + { + "name": "http_set_cookie", + "label": "HTTP.Set Cookie", + "type": "string" + }, + { + "name": "http_version", + "label": "HTTP.Version", + "type": "string" + }, + { + "name": "http_response_latency_ms", + "label": "HTTP.Response Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "http_session_duration_ms", + "label": "HTTP.Session Duration(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "http_action_file_size", + "label": "HTTP.Action File Size", + "type": "int" + }, + { + "name": "mail_protocol_type", + "label": "Mail.Protocol Type", + "type": "string" + }, + { + "name": "mail_account", + "label": "Mail.Account", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "mail_from_cmd", + "label": "Mail.From CMD", + "type": "string" + }, + { + "name": "mail_to_cmd", + "label": "Mail.To CMD", + "type": "string" + }, + { + "name": "mail_from", + "label": "Mail.From", + "doc": { + "allow_query": "true", + "constraints": { + "type": "email" + } + }, + "type": "string" + }, + { + "name": "mail_to", + "label": "Mail.To", + "doc": { + "allow_query": "true", + "constraints": { + "type": "email" + } + }, + "type": "string" + }, + { + "name": "mail_cc", + "label": "Mail.CC", + "type": "string" + }, + { + "name": "mail_bcc", + "label": "Mail.BCC", + "type": "string" + }, + { + "name": "mail_subject", + "label": "Mail.Subject", + "doc": { + "allow_query": "true", + "format": { + "functions": "decode_of_base64", + "param": "$.mail_subject_charset" + } + }, + "type": "string" + }, + { + "name": "mail_subject_charset", + "label": "Mail.Subject Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_content", + "label": "Mail.Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_content_charset", + "label": "Mail.Content Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_attachment_name", + "label": "Mail.Attachment", + "doc": { + "format": { + "functions": "decode_of_base64", + "param": "$.mail_attachment_name_charset" + } + }, + "type": "string" + }, + { + "name": "mail_attachment_name_charset", + "label": "Mail.Attachment Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_attachment_content", + "label": "Mail.Attachment Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_eml_file", + "label": "Mail.EML File", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "mail_snapshot", + "label": "Mail.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "dns_message_id", + "label": "DNS.Message ID", + "type": "int" + }, + { + "name": "dns_qr", + "label": "DNS.QR", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "RESPONSE" + } + ] + }, + "type": "int" + }, + { + "name": "dns_opcode", + "label": "DNS.OPCODE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "IQUERY" + }, + { + "code": "2", + "value": "STATUS" + }, + { + "code": "5", + "value": "UPDATE" + } + ] + }, + "type": "int" + }, + { + "name": "dns_aa", + "label": "DNS.AA", + "type": "int" + }, + { + "name": "dns_tc", + "label": "DNS.TC", + "type": "int" + }, + { + "name": "dns_rd", + "label": "DNS.RD", + "type": "int" + }, + { + "name": "dns_ra", + "label": "DNS.RA", + "type": "int" + }, + { + "name": "dns_rcode", + "label": "DNS.RCODE", + "type": "int" + }, + { + "name": "dns_qdcount", + "label": "DNS.QDCOUNT", + "type": "int" + }, + { + "name": "dns_ancount", + "label": "DNS.ANCOUNT", + "type": "int" + }, + { + "name": "dns_nscount", + "label": "DNS.NSCOUNT", + "type": "int" + }, + { + "name": "dns_arcount", + "label": "DNS.ARCOUNT", + "type": "int" + }, + { + "name": "dns_qname", + "label": "DNS.QNAME", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "dns_qtype", + "label": "DNS.QTYPE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "A" + }, + { + "code": "2", + "value": "NS" + }, + { + "code": "5", + "value": "CNAME" + }, + { + "code": "6", + "value": "SOA" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "12", + "value": "PTR" + }, + { + "code": "13", + "value": "HINFO" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "15", + "value": "MX" + }, + { + "code": "28", + "value": "AAAA" + } + ] + }, + "type": "int" + }, + { + "name": "dns_qclass", + "label": "DNS.QCLASS", + "type": "int" + }, + { + "name": "dns_cname", + "label": "DNS.CNAME", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "dns_sub", + "label": "DNS.SUB", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "DNS" + }, + { + "code": "2", + "value": "DNSSEC" + } + ] + }, + "type": "int" + }, + { + "name": "dns_rr", + "label": "DNS.RR", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_version", + "label": "SSL.Version", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_sni", + "label": "SSL.SNI", + "doc": { + "allow_query": "true", + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "ssl_san", + "label": "SSL.SAN", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_cn", + "label": "SSL.CN", + "type": "string" + }, + { + "name": "ssl_pinningst", + "label": "SSL.Pinning", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Not Pinning" + }, + { + "code": "1", + "value": "Pinning" + }, + { + "code": "2", + "value": "Maybe Pinning" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_intercept_state", + "label": "SSL.Intercept State", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Passthrough" + }, + { + "code": "1", + "value": "Intercept" + }, + { + "code": "2", + "value": "Shutdown" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_server_side_latency", + "label": "SSL.Server Side Latency(ms)", + "type": "int" + }, + { + "name": "ssl_client_side_latency", + "label": "SSL.Client Side Latency(ms)", + "type": "int" + }, + { + "name": "ssl_server_side_version", + "label": "SSL.Server Side Version", + "type": "string" + }, + { + "name": "ssl_client_side_version", + "label": "SSL.Client Side Version", + "type": "string" + }, + { + "name": "ssl_cert_verify", + "label": "SSL.Certificate Verify", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_error", + "label": "SSL.Error", + "type": "string" + }, + { + "name": "ssl_con_latency_ms", + "label": "SSL.Connection Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "ssl_ja3_fingerprint", + "label": "SSL.JA3", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "ssl_ja3_hash", + "label": "SSL.JA3 hash", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ssl_cert_issuer", + "label": "SSL.Issuer", + "doc": { + "constraints": { + "type": "items" + } + }, + "type": "string" + }, + { + "name": "ssl_cert_subject", + "label": "SSL.Subject", + "doc": { + "constraints": { + "type": "items" + } + }, + "type": "string" + }, + { + "name": "quic_version", + "label": "QUIC.Version", + "type": "string" + }, + { + "name": "quic_sni", + "label": "QUIC.SNI", + "doc": { + "allow_query": "true", + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "quic_user_agent", + "label": "QUIC.User Agent", + "type": "string" + }, + { + "name": "ftp_account", + "label": "FTP.Account", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ftp_url", + "label": "FTP.URL", + "type": "string" + }, + { + "name": "ftp_content", + "label": "FTP.Content", + "type": "string" + }, + { + "name": "ftp_link_type", + "label": "FTP.Link Type", + "type": "string" + }, + { + "name": "bgp_type", + "label": "BGP.Type", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "bgp_as_num", + "label": "BGP.AS Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "bgp_route", + "label": "BGP.Route", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_calling_account", + "label": "VoIP.Calling Account", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_called_account", + "label": "VoIP.Called Account", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_calling_number", + "label": "VoIP.Calling Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_called_number", + "label": "VoIP.Called Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "streaming_media_url", + "label": "Streaming.Media URL", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "streaming_media_protocol", + "label": "Streaming.Media Protocol", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "app_extra_info", + "label": "APP.Extra Info", + "type": "string" + }, + { + "name": "sip_call_id", + "label": "SIP.Call-ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "sip_originator_description", + "label": "SIP.Originator", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "sip_responder_description", + "label": "SIP.Responder", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "sip_user_agent", + "label": "SIP.User-Agent", + "type": "string" + }, + { + "name": "sip_server", + "label": "SIP.Server", + "type": "string" + }, + { + "name": "sip_originator_sdp_connect_ip", + "label": "SIP.Originator IP", + "type": "string" + }, + { + "name": "sip_originator_sdp_media_port", + "label": "SIP.Originator Port", + "type": "int" + }, + { + "name": "sip_originator_sdp_media_type", + "label": "SIP.Originator Media Type", + "type": "string" + }, + { + "name": "sip_originator_sdp_content", + "label": "SIP.Originator Content", + "type": "string" + }, + { + "name": "sip_responder_sdp_connect_ip", + "label": "SIP.Responder IP", + "type": "string" + }, + { + "name": "sip_responder_sdp_media_port", + "label": "SIP.Responder Port", + "type": "int" + }, + { + "name": "sip_responder_sdp_media_type", + "label": "SIP.Responder Media Type", + "type": "string" + }, + { + "name": "sip_responder_sdp_content", + "label": "SIP.Responder Content", + "type": "string" + }, + { + "name": "sip_duration", + "label": "SIP.Duration", + "type": "int" + }, + { + "name": "sip_bye", + "label": "SIP.Bye", + "type": "string" + }, + { + "name": "rtp_payload_type_c2s", + "label": "RTP.Payload Type(c2s)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_payload_type_s2c", + "label": "RTP.Payload Type(s2c)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_pcap_path", + "label": "RTP.PCAP", + "doc": { + "constraints": { + "type": "files" + } + }, + "type": "string" + }, + { + "name": "rtp_originator_dir", + "label": "RTP.Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "unknown" + }, + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + } + ] + }, + "type": "int" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/meta_data.json b/parcels/roles/services/galaxy-qgw-service/files/config/meta_data.json new file mode 100644 index 0000000..da93253 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/meta_data.json @@ -0,0 +1,73 @@ +[ + { + "namespace":"tsg_galaxy_v3", + "group":"CLICKHOUSE_GROUP", + "tables":[ + "connection_record_log", + "proxy_ip_info", + "radius_record_log", + "security_ip_info", + "connection_record_log_common_subscriber_id", + "proxy_event_log", + "security_event_log", + "connection_record_log_http_domain", + "sys_packet_capture_log", + "radius_onff_log", + "active_defence_event_log", + "connection_record_log_common_server_ip", + "security_website_domain_info", + "connection_record_log_common_client_ip", + "voip_record_log", + "transaction_record_log", + "gtpc_record_log", + "live_session_record_log" + ] + }, + { + "namespace":"system", + "group":"CLICKHOUSE_GROUP", + "tables":[ + "query_log_cluster", + "tables_cluster", + "columns_cluster", + "disks_cluster", + "parts_cluster", + "processes", + "query_log" + ] + }, + { + "namespace":"druid", + "group":"DRUID_GROUP", + "tables":[ + "top_domain_recommend_daily_log", + "source_country_ip_num_log", + "top_internal_host_log", + "top_website_domain_log", + "sd_multi_access_cnt_feature", + "proxy_event_hits_log", + "sys_storage_log", + "domain_country_num_log", + "security_event_hits_log", + "traffic_protocol_stat_log", + "top_website_urls_daily_log", + "top_server_ip_log", + "traffic_summary_log", + "traffic_metrics_log", + "top_user_log", + "destination_country_log", + "top_urls_log", + "top_client_ip_log", + "top_external_host_log", + "traffic_app_stat_log" + ] + }, + { + "namespace":"etl", + "group":"ETL_GROUP", + "tables":[ + "liveChart_history", + "liveChart_live" + ] + } +]
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/parts_cluster.json b/parcels/roles/services/galaxy-qgw-service/files/config/parts_cluster.json new file mode 100644 index 0000000..c311abf --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/parts_cluster.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "parts_cluster", + "fields": [ + { + "name": "name", + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/processes.json b/parcels/roles/services/galaxy-qgw-service/files/config/processes.json new file mode 100644 index 0000000..75d74a9 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/processes.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "processes", + "fields": [ + { + "name": "query_id", + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/proxy_event_hits_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/proxy_event_hits_log.json new file mode 100644 index 0000000..5e3ff8a --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/proxy_event_hits_log.json @@ -0,0 +1,58 @@ +{ + "type": "record", + "name": "proxy_event_hits_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "action", + "type": "long" + }, + { + "name": "isp", + "type": "string" + }, + { + "name": "entrance_id", + "type": "long" + }, + { + "name": "hits", + "type": "long" + }, + { + "name": "policy_id", + "type": "long" + }, + { + "name": "sub_action", + "type": "string" + }, + { + "name": "country", + "type": "string" + }, + { + "name": "location", + "type": "string" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "ip_object", + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/proxy_event_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/proxy_event_log.json new file mode 100644 index 0000000..0f0072b --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/proxy_event_log.json @@ -0,0 +1,2025 @@ +{ + "type": "record", + "name": "proxy_event_log", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "functions": { + "aggregation": [ + { + "name": "COUNT", + "label": "COUNT", + "function": "count(expr)" + }, + { + "name": "COUNT_DISTINCT", + "label": "COUNT_DISTINCT", + "function": "count(distinct expr)" + }, + { + "name": "AVG", + "label": "AVG", + "function": "avg(expr)" + }, + { + "name": "SUM", + "label": "SUM", + "function": "sum(expr)" + }, + { + "name": "MAX", + "label": "MAX", + "function": "max(expr)" + }, + { + "name": "MIN", + "label": "MIN", + "function": "min(expr)" + } + ], + "operator": [ + { + "name": "=", + "label": "=", + "function": "expr = value" + }, + { + "name": "!=", + "label": "!=", + "function": "expr != value" + }, + { + "name": ">", + "label": ">", + "function": "expr > value" + }, + { + "name": "<", + "label": "<", + "function": "expr < value" + }, + { + "name": ">=", + "label": ">=", + "function": "expr >= value" + }, + { + "name": "<=", + "label": "<=", + "function": "expr <= value" + }, + { + "name": "has", + "label": "HAS", + "function": "has(expr, value)" + }, + { + "name": "in", + "label": "IN", + "function": "expr in (values)" + }, + { + "name": "not in", + "label": "NOT IN", + "function": "expr not in (values)" + }, + { + "name": "like", + "label": "LIKE", + "function": "expr like value" + }, + { + "name": "not like", + "label": "NOT LIKE", + "function": "expr not like value" + }, + { + "name": "notEmpty", + "label": "NOT EMPTY", + "function": "notEmpty(expr)" + }, + { + "name": "empty", + "label": "EMPTY", + "function": "empty(expr)" + } + ] + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_policy_id", + "common_sub_action", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_data_center", + "common_client_asn", + "common_server_asn", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url", + "doh_host", + "doh_qname" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_subscriber_id", + "common_sled_ip", + "common_device_id", + "common_sessions", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url", + "doh_host", + "doh_qname" + ], + "filters": [ + "common_policy_id", + "common_sub_action", + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_l4_protocol", + "common_data_center", + "common_sled_ip", + "common_device_id", + "common_client_asn", + "common_server_asn", + "common_direction", + "common_schema_type", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url", + "http_content_type", + "doh_host", + "doh_qname" + ], + "references": { + "aggregation": [ + { + "type": "int", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "long", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "float", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "double", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "string", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "date", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + } + ], + "operator": [ + { + "type": "int", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "long", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "float", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "double", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "string", + "functions": "=,!=,in,not in,like,not like,notEmpty,empty" + }, + { + "type": "date", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "timestamp", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "array", + "functions": "has" + } + ] + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "HTTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "http_url", + "http_host", + "http_domain", + "http_request_line", + "http_response_line", + "http_request_header", + "http_response_header", + "http_request_content", + "http_response_content", + "http_request_body", + "http_response_body", + "http_request_body_key", + "http_response_body_key", + "http_proxy_flag", + "http_sequence", + "http_snapshot", + "http_cookie", + "http_referer", + "http_user_agent", + "http_content_length", + "http_content_type", + "http_set_cookie", + "http_version", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "http_url", + "common_sub_action" + ] + }, + "DoH": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "doh_url", + "doh_host", + "doh_request_line", + "doh_response_line", + "doh_cookie", + "doh_referer", + "doh_user_agent", + "doh_content_length", + "doh_content_type", + "doh_set_cookie", + "doh_version", + "doh_message_id", + "doh_qr", + "doh_opcode", + "doh_aa", + "doh_tc", + "doh_rd", + "doh_ra", + "doh_rcode", + "doh_qdcount", + "doh_ancount", + "doh_nscount", + "doh_arcount", + "doh_qname", + "doh_qtype", + "doh_qclass", + "doh_cname", + "doh_sub", + "doh_rr" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_client_ip", + "doh_url", + "doh_qname", + "common_server_port" + ] + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "common_sub_action", + "common_schema_type" + ], + "tunnel_type": { + "GTP": [ + { + "name": "gtp_sgw_ip", + "label": "S-GW IP", + "type": "string" + }, + { + "name": "gtp_pgw_ip", + "label": "P-GW IP", + "type": "string" + }, + { + "name": "gtp_sgw_port", + "label": "S-GW Port", + "type": "int" + }, + { + "name": "gtp_pgw_port", + "label": "P-GW Port", + "type": "int" + }, + { + "name": "gtp_uplink_teid", + "label": "Uplink TEID", + "type": "long" + }, + { + "name": "gtp_downlink_teid", + "label": "Downlink TEID", + "type": "long" + } + ], + "MPLS": [ + { + "name": "mpls_c2s_direction_label", + "label": "Multiprotocol Label(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "mpls_s2c_direction_label", + "label": "Multiprotocol Label(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "VLAN": [ + { + "name": "vlan_c2s_direction_id", + "label": "VLAN Direction(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "vlan_s2c_direction_id", + "label": "VLAN Direction(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "ETHERNET": [ + { + "name": "source_mac", + "label": "Source MAC", + "type": "string" + }, + { + "name": "destination_mac", + "label": "Destination MAC", + "type": "string" + } + ], + "MULTIPATH_ETHERNET": [ + { + "name": "c2s_source_mac", + "label": "Source MAC(c2s)", + "type": "string" + }, + { + "name": "c2s_destination_mac", + "label": "Destination MAC(c2s)", + "type": "string" + }, + { + "name": "s2c_source_mac", + "label": "Source MAC(s2c)", + "type": "string" + }, + { + "name": "s2c_destination_mac", + "label": "Destination MAC(s2c)", + "type": "string" + } + ] + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn,radius_match", + "appendTo": "common_client_asn,common_subscriber_id" + } + }, + "type": "string" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "type": "int", + "doc": { + "allow_query": "true" + } + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "allow_query": "true", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "48", + "value": "Manipulation" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "doc": { + "format": { + "functions": "set_value", + "param": "1" + } + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "allow_query": "true", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "DoH", + "value": "DoH" + } + ] + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "City A", + "value": "City A" + }, + { + "code": "City B", + "value": "City B" + }, + { + "code": "City C", + "value": "City C" + }, + { + "code": "City D", + "value": "City D" + }, + { + "code": "City E", + "value": "City E" + }, + { + "code": "City F", + "value": "City F" + }, + { + "code": "City G", + "value": "City G" + }, + { + "code": "City H", + "value": "City H" + }, + { + "code": "City I", + "value": "City I" + }, + { + "code": "City J", + "value": "City J" + }, + { + "code": "City K", + "value": "City K" + }, + { + "code": "City L", + "value": "City L" + }, + { + "code": "City M", + "value": "City M" + }, + { + "code": "City N", + "value": "City N" + }, + { + "code": "City O", + "value": "City O" + }, + { + "code": "City P", + "value": "City P" + }, + { + "code": "City Q", + "value": "City Q" + }, + { + "code": "City R", + "value": "City R" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Ethernet" + }, + { + "code": "8", + "value": "PPP" + }, + { + "code": "12", + "value": "CiscoHDLC" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "allow_query": "true", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ] + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "http_url", + "label": "HTTP.URL", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_host", + "label": "HTTP.Host", + "doc": { + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "http_domain", + "label": "HTTP.Domain", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_request_line", + "label": "HTTP.Request Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_line", + "doc": { + "visibility": "disabled" + }, + "label": "HTTP.Response Line", + "type": "string" + }, + { + "name": "http_request_header", + "label": "HTTP.Request Header", + "type": "string" + }, + { + "name": "http_response_header", + "label": "HTTP.Response Header", + "type": "string" + }, + { + "name": "http_request_content", + "label": "HTTP.Request Content", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_content", + "label": "HTTP.Response Content", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_request_body", + "label": "HTTP.Request Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_response_body", + "label": "HTTP.Response Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_request_body_key", + "label": "HTTP.Request Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_body_key", + "label": "HTTP.Response Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_proxy_flag", + "label": "HTTP.Proxy Flag", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_sequence", + "label": "HTTP.Sequence", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_snapshot", + "label": "HTTP.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_cookie", + "label": "HTTP.Cookie", + "type": "string" + }, + { + "name": "http_referer", + "label": "HTTP.Referer", + "type": "string" + }, + { + "name": "http_user_agent", + "label": "HTTP.User Agent", + "type": "string" + }, + { + "name": "http_content_length", + "label": "HTTP.Content Length", + "type": "string" + }, + { + "name": "http_content_type", + "label": "HTTP.Content Type", + "type": "string" + }, + { + "name": "http_set_cookie", + "label": "HTTP.Set Cookie", + "type": "string" + }, + { + "name": "http_version", + "label": "HTTP.Version", + "type": "string" + }, + { + "name": "http_response_latency_ms", + "label": "HTTP.Response Latency(ms)", + "type": "long" + }, + { + "name": "http_session_duration_ms", + "label": "HTTP.Session Duration(ms)", + "type": "long" + }, + { + "name": "http_action_file_size", + "label": "HTTP.Action File Size", + "type": "int" + }, + { + "name": "doh_url", + "label": "DoH.URL", + "type": "string" + }, + { + "name": "doh_host", + "label": "DoH.Host", + "type": "string" + }, + { + "name": "doh_request_line", + "label": "DoH.Request Line", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "doh_response_line", + "doc": { + "visibility": "hidden" + }, + "label": "DoH.Response Line", + "type": "string" + }, + { + "name": "doh_cookie", + "label": "DoH.Cookie", + "type": "string" + }, + { + "name": "doh_referer", + "label": "DoH.Referer", + "type": "string" + }, + { + "name": "doh_user_agent", + "label": "DoH.User Agent", + "type": "string" + }, + { + "name": "doh_content_length", + "label": "DoH.Content Length", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "doh_content_type", + "label": "DoH.Content Type", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "doh_set_cookie", + "label": "DoH.Set Cookie", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "doh_version", + "label": "DoH.Version", + "type": "string" + }, + { + "name": "doh_message_id", + "label": "DoH.Message ID", + "type": "int" + }, + { + "name": "doh_qr", + "label": "DoH.QR", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "REESPONSE" + } + ] + }, + "type": "int" + }, + { + "name": "doh_opcode", + "label": "DoH.OPCODE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "IQUERY" + }, + { + "code": "2", + "value": "STATUS" + }, + { + "code": "5", + "value": "UPDATE" + } + ] + }, + "type": "int" + }, + { + "name": "doh_aa", + "label": "DoH.AA", + "type": "int" + }, + { + "name": "doh_tc", + "label": "DoH.TC", + "type": "int" + }, + { + "name": "doh_rd", + "label": "DoH.RD", + "type": "int" + }, + { + "name": "doh_ra", + "label": "DoH.RA", + "type": "int" + }, + { + "name": "doh_rcode", + "label": "DoH.RCODE", + "type": "int" + }, + { + "name": "doh_qdcount", + "label": "DoH.QDCOUNT", + "type": "int" + }, + { + "name": "doh_ancount", + "label": "DoH.ANCOUNT", + "type": "int" + }, + { + "name": "doh_nscount", + "label": "DoH.NSCOUNT", + "type": "int" + }, + { + "name": "doh_arcount", + "label": "DoH.ARCOUNT", + "type": "int" + }, + { + "name": "doh_qname", + "label": "DoH.QNAME", + "type": "string" + }, + { + "name": "doh_qtype", + "label": "DoH.QTYPE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "A" + }, + { + "code": "2", + "value": "NS" + }, + { + "code": "5", + "value": "CNAME" + }, + { + "code": "6", + "value": "SOA" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "12", + "value": "PTR" + }, + { + "code": "13", + "value": "HINFO" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "15", + "value": "MX" + }, + { + "code": "28", + "value": "AAAA" + } + ] + }, + "type": "int" + }, + { + "name": "doh_qclass", + "label": "DoH.QCLASS", + "type": "int" + }, + { + "name": "doh_cname", + "label": "DoH.CNAME", + "type": "string" + }, + { + "name": "doh_sub", + "label": "DoH.SUB", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "DNS" + }, + { + "code": "2", + "value": "DNSSEC" + } + ] + }, + "type": "int" + }, + { + "name": "doh_rr", + "label": "DoH.RR", + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/query_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/query_log.json new file mode 100644 index 0000000..4f5e8d5 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/query_log.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "query_log", + "fields": [ + { + "name": "query_id", + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/query_log_cluster.json b/parcels/roles/services/galaxy-qgw-service/files/config/query_log_cluster.json new file mode 100644 index 0000000..d6e7583 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/query_log_cluster.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "query_log_cluster", + "fields": [ + { + "name": "type", + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/radius_onff_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/radius_onff_log.json new file mode 100644 index 0000000..9201ebb --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/radius_onff_log.json @@ -0,0 +1,37 @@ +{ + "type": "record", + "name": "radius_onff_log", + "namespace": "tsg_galaxy_v3", + "fields": [ + { + "name": "event_timestamp", + "label": "Event Time", + "type": "long" + }, + { + "name": "account", + "label": "Account", + "type": "string" + }, + { + "name": "framed_ip", + "label": "Framed IP", + "type": "string" + }, + { + "name": "acct_session_id", + "label": "Acct Session ID", + "type": "string" + }, + { + "name": "acct_status_type", + "label": "Acct Status Type", + "type": "int" + }, + { + "name": "acct_session_time", + "label": "Acct Session Time", + "type": "int" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/radius_record_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/radius_record_log.json new file mode 100644 index 0000000..122f454 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/radius_record_log.json @@ -0,0 +1,1682 @@ +{ + "type": "record", + "name": "radius_record_log", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "functions": { + "aggregation": [ + { + "name": "COUNT", + "label": "COUNT", + "function": "count(expr)" + }, + { + "name": "COUNT_DISTINCT", + "label": "COUNT_DISTINCT", + "function": "count(distinct expr)" + }, + { + "name": "AVG", + "label": "AVG", + "function": "avg(expr)" + }, + { + "name": "SUM", + "label": "SUM", + "function": "sum(expr)" + }, + { + "name": "MAX", + "label": "MAX", + "function": "max(expr)" + }, + { + "name": "MIN", + "label": "MIN", + "function": "min(expr)" + } + ], + "operator": [ + { + "name": "=", + "label": "=", + "function": "expr = value" + }, + { + "name": "!=", + "label": "!=", + "function": "expr != value" + }, + { + "name": ">", + "label": ">", + "function": "expr > value" + }, + { + "name": "<", + "label": "<", + "function": "expr < value" + }, + { + "name": ">=", + "label": ">=", + "function": "expr >= value" + }, + { + "name": "<=", + "label": "<=", + "function": "expr <= value" + }, + { + "name": "has", + "label": "HAS", + "function": "has(expr, value)" + }, + { + "name": "in", + "label": "IN", + "function": "expr in (values)" + }, + { + "name": "not in", + "label": "NOT IN", + "function": "expr not in (values)" + }, + { + "name": "like", + "label": "LIKE", + "function": "expr like value" + }, + { + "name": "not like", + "label": "NOT LIKE", + "function": "expr not like value" + }, + { + "name": "notEmpty", + "label": "NOT EMPTY", + "function": "notEmpty(expr)" + }, + { + "name": "empty", + "label": "EMPTY", + "function": "empty(expr)" + } + ] + }, + "schema_query": { + "dimensions": [ + "radius_nas_ip", + "radius_framed_ip", + "common_subscriber_id" + ], + "metrics": [ + "radius_framed_ip", + "radius_event_timestamp", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num" + ], + "filters": [ + "radius_framed_ip", + "common_subscriber_id", + "radius_packet_type", + "radius_acct_session_id", + "radius_acct_multi_session_id", + "radius_acct_status_type" + ], + "references": { + "aggregation": [ + { + "type": "int", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "long", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "float", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "double", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "string", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "date", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + } + ], + "operator": [ + { + "type": "int", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "long", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "float", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "double", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "string", + "functions": "=,!=,in,not in,like,not like,notEmpty,empty" + }, + { + "type": "date", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "timestamp", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "array", + "functions": "has" + } + ] + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "RADIUS": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "radius_packet_type", + "radius_nas_ip", + "radius_framed_ip", + "radius_account", + "radius_session_timeout", + "radius_idle_timeout", + "radius_acct_status_type", + "radius_acct_terminate_cause", + "radius_event_timestamp", + "radius_nas_port", + "radius_service_type", + "radius_framed_protocol", + "radius_callback_number", + "radius_callback_id", + "radius_termination_action", + "radius_called_station_id", + "radius_calling_station_id", + "radius_acct_delay_time", + "radius_acct_session_id", + "radius_acct_multi_session_id", + "radius_acct_input_octets", + "radius_acct_output_octets", + "radius_acct_input_packets", + "radius_acct_output_packets", + "radius_acct_session_time", + "radius_acct_link_count", + "radius_acct_interim_interval" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "radius_nas_ip", + "radius_framed_ip", + "radius_acct_status_type" + ] + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "radius_nas_ip", + "radius_framed_ip", + "radius_acct_status_type" + ], + "tunnel_type": { + "GTP": [ + { + "name": "gtp_sgw_ip", + "label": "S-GW IP", + "type": "string" + }, + { + "name": "gtp_pgw_ip", + "label": "P-GW IP", + "type": "string" + }, + { + "name": "gtp_sgw_port", + "label": "S-GW Port", + "type": "int" + }, + { + "name": "gtp_pgw_port", + "label": "P-GW Port", + "type": "int" + }, + { + "name": "gtp_uplink_teid", + "label": "Uplink TEID", + "type": "long" + }, + { + "name": "gtp_downlink_teid", + "label": "Downlink TEID", + "type": "long" + } + ], + "MPLS": [ + { + "name": "mpls_c2s_direction_label", + "label": "Multiprotocol Label(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "mpls_s2c_direction_label", + "label": "Multiprotocol Label(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "VLAN": [ + { + "name": "vlan_c2s_direction_id", + "label": "VLAN Direction(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "vlan_s2c_direction_id", + "label": "VLAN Direction(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "ETHERNET": [ + { + "name": "source_mac", + "label": "Source MAC", + "type": "string" + }, + { + "name": "destination_mac", + "label": "Destination MAC", + "type": "string" + } + ], + "MULTIPATH_ETHERNET": [ + { + "name": "c2s_source_mac", + "label": "Source MAC(c2s)", + "type": "string" + }, + { + "name": "c2s_destination_mac", + "label": "Destination MAC(c2s)", + "type": "string" + }, + { + "name": "s2c_source_mac", + "label": "Source MAC(s2c)", + "type": "string" + }, + { + "name": "s2c_destination_mac", + "label": "Destination MAC(s2c)", + "type": "string" + } + ] + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "visibility": "hidden", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "48", + "value": "Manipulation" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "BASE", + "value": "BASE" + }, + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "MAIL", + "value": "MAIL" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "SSL", + "value": "SSL" + }, + { + "code": "FTP", + "value": "FTP" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "City A", + "value": "City A" + }, + { + "code": "City B", + "value": "City B" + }, + { + "code": "City C", + "value": "City C" + }, + { + "code": "City D", + "value": "City D" + }, + { + "code": "City E", + "value": "City E" + }, + { + "code": "City F", + "value": "City F" + }, + { + "code": "City G", + "value": "City G" + }, + { + "code": "City H", + "value": "City H" + }, + { + "code": "City I", + "value": "City I" + }, + { + "code": "City J", + "value": "City J" + }, + { + "code": "City K", + "value": "City K" + }, + { + "code": "City L", + "value": "City L" + }, + { + "code": "City M", + "value": "City M" + }, + { + "code": "City N", + "value": "City N" + }, + { + "code": "City O", + "value": "City O" + }, + { + "code": "City P", + "value": "City P" + }, + { + "code": "City Q", + "value": "City Q" + }, + { + "code": "City R", + "value": "City R" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Ethernet" + }, + { + "code": "8", + "value": "PPP" + }, + { + "code": "12", + "value": "CiscoHDLC" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "visibility": "disabled", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + }, + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ] + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "radius_packet_type", + "label": "Packet Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "Access-Request" + }, + { + "code": "2", + "value": "Access-Accept" + }, + { + "code": "3", + "value": "Access-Reject" + }, + { + "code": "4", + "value": "Accounting-Request" + }, + { + "code": "5", + "value": "Accounting-Response" + }, + { + "code": "11", + "value": "Access-Challenge" + } + ] + }, + "type": "int" + }, + { + "name": "radius_account", + "label": "Account", + "doc": { + "format": { + "functions": "get_value", + "appendTo": "common_subscriber_id" + } + }, + "type": "string" + }, + { + "name": "radius_nas_ip", + "label": "Nas IP", + "type": "string" + }, + { + "name": "radius_framed_ip", + "label": "Framed IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "radius_session_timeout", + "label": "Session Timeout", + "type": "int" + }, + { + "name": "radius_idle_timeout", + "label": "Idle Timeout", + "type": "int" + }, + { + "name": "radius_acct_status_type", + "label": "ACC Status Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "Start" + }, + { + "code": "2", + "value": "Stop" + }, + { + "code": "3", + "value": "Interim-Update" + }, + { + "code": "7", + "value": "Accounting-On" + }, + { + "code": "8", + "value": "Accounting-Off" + } + ] + }, + "type": "int" + }, + { + "name": "radius_acct_terminate_cause", + "label": "Acct Terminate Cause", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "User Request" + }, + { + "code": "2", + "value": "Lost Carrier" + }, + { + "code": "3", + "value": "Lost Service" + }, + { + "code": "4", + "value": "Idle Timeout" + }, + { + "code": "5", + "value": "Session Timeout" + }, + { + "code": "6", + "value": "Admin Reset" + }, + { + "code": "7", + "value": "Admin Reboot" + }, + { + "code": "8", + "value": "Port Error" + }, + { + "code": "9", + "value": "NAS Error" + }, + { + "code": "10", + "value": "NAS Request" + }, + { + "code": "11", + "value": "NAS Reboot" + }, + { + "code": "12", + "value": "Port Unneeded" + }, + { + "code": "13", + "value": "Port Preempted" + }, + { + "code": "14", + "value": "Port Suspended" + }, + { + "code": "15", + "value": "Service Unavailable" + }, + { + "code": "16", + "value": "Callback" + }, + { + "code": "17", + "value": "User Error" + }, + { + "code": "18", + "value": "Host Request" + } + ] + }, + "type": "int" + }, + { + "name": "radius_event_timestamp", + "label": "Event Timestamp", + "type": "int" + }, + { + "name": "radius_service_type", + "label": "Service Type", + "type": "int" + }, + { + "name": "radius_nas_port", + "label": "Nas Port", + "type": "int" + }, + { + "name": "radius_framed_protocol", + "label": "Framed Protocol", + "type": "int" + }, + { + "name": "radius_callback_number", + "label": "Callback Number", + "type": "string" + }, + { + "name": "radius_callback_id", + "label": "Callback ID", + "type": "string" + }, + { + "name": "radius_termination_action", + "label": "Termination Action", + "type": "int" + }, + { + "name": "radius_called_station_id", + "label": "Called Station ID", + "type": "string" + }, + { + "name": "radius_calling_station_id", + "label": "Calling Station ID", + "type": "string" + }, + { + "name": "radius_acct_delay_time", + "label": "Acct Delay Time", + "type": "int" + }, + { + "name": "radius_acct_session_id", + "label": "Acct Session ID", + "type": "string" + }, + { + "name": "radius_acct_multi_session_id", + "label": "Acct Multi Session ID", + "type": "string" + }, + { + "name": "radius_acct_input_octets", + "label": "Acct Input Octets", + "type": "long" + }, + { + "name": "radius_acct_output_octets", + "label": "Acct Output Octets", + "type": "long" + }, + { + "name": "radius_acct_input_packets", + "label": "Acct Input Packets", + "type": "long" + }, + { + "name": "radius_acct_output_packets", + "label": "Acct Output Packets", + "type": "long" + }, + { + "name": "radius_acct_session_time", + "label": "Acct Session Time", + "type": "int" + }, + { + "name": "radius_acct_link_count", + "label": "Acct Link Count", + "type": "int" + }, + { + "name": "radius_acct_interim_interval", + "label": "Acct Interim Interval", + "type": "int" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/security_event_hits_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/security_event_hits_log.json new file mode 100644 index 0000000..c445244 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/security_event_hits_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "security_event_hits_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "action", + "type": "long" + }, + { + "name": "entrance_id", + "type": "long" + }, + { + "name": "hits", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "isp", + "type": "string" + }, + { + "name": "policy_id", + "type": "long" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/security_event_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/security_event_log.json new file mode 100644 index 0000000..c5d34cd --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/security_event_log.json @@ -0,0 +1,3682 @@ +{ + "type": "record", + "name": "security_event_log", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "functions": { + "aggregation": [ + { + "name": "COUNT", + "label": "COUNT", + "function": "count(expr)" + }, + { + "name": "COUNT_DISTINCT", + "label": "COUNT_DISTINCT", + "function": "count(distinct expr)" + }, + { + "name": "AVG", + "label": "AVG", + "function": "avg(expr)" + }, + { + "name": "SUM", + "label": "SUM", + "function": "sum(expr)" + }, + { + "name": "MAX", + "label": "MAX", + "function": "max(expr)" + }, + { + "name": "MIN", + "label": "MIN", + "function": "min(expr)" + } + ], + "operator": [ + { + "name": "=", + "label": "=", + "function": "expr = value" + }, + { + "name": "!=", + "label": "!=", + "function": "expr != value" + }, + { + "name": ">", + "label": ">", + "function": "expr > value" + }, + { + "name": "<", + "label": "<", + "function": "expr < value" + }, + { + "name": ">=", + "label": ">=", + "function": "expr >= value" + }, + { + "name": "<=", + "label": "<=", + "function": "expr <= value" + }, + { + "name": "has", + "label": "HAS", + "function": "has(expr, value)" + }, + { + "name": "in", + "label": "IN", + "function": "expr in (values)" + }, + { + "name": "not in", + "label": "NOT IN", + "function": "expr not in (values)" + }, + { + "name": "like", + "label": "LIKE", + "function": "expr like value" + }, + { + "name": "not like", + "label": "NOT LIKE", + "function": "expr not like value" + }, + { + "name": "notEmpty", + "label": "NOT EMPTY", + "function": "notEmpty(expr)" + }, + { + "name": "empty", + "label": "EMPTY", + "function": "empty(expr)" + } + ] + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_policy_id", + "common_action", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_l4_protocol", + "common_l7_protocol", + "common_data_center", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "ssl_sni", + "ssl_ja3_hash", + "ssl_client_side_version", + "ssl_server_side_version", + "mail_account", + "mail_from", + "mail_to", + "quic_sni" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_subscriber_id", + "common_sled_ip", + "common_device_id", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "ssl_sni", + "ssl_ja3_hash", + "ssl_client_side_latency", + "ssl_server_side_latency", + "mail_account", + "mail_from", + "mail_to", + "quic_sni" + ], + "filters": [ + "common_policy_id", + "common_action", + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_l4_protocol", + "common_l7_protocol", + "common_stream_dir", + "common_data_center", + "common_sled_ip", + "common_device_id", + "common_direction", + "common_schema_type", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_imei", + "common_imsi", + "common_phone_number", + "common_app_label", + "http_host", + "http_domain", + "http_url", + "http_content_type", + "ssl_sni", + "ssl_ja3_hash", + "ssl_pinningst", + "ssl_intercept_state", + "ssl_client_side_version", + "ssl_server_side_version", + "ssl_cert_verify", + "ssl_client_side_latency", + "ssl_server_side_latency", + "mail_account", + "mail_from", + "mail_to", + "mail_subject", + "quic_sni" + ], + "references": { + "aggregation": [ + { + "type": "int", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "long", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "float", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "double", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "string", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "date", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + } + ], + "operator": [ + { + "type": "int", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "long", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "float", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "double", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "string", + "functions": "=,!=,in,not in,like,not like,notEmpty,empty" + }, + { + "type": "date", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "timestamp", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "array", + "functions": "has" + } + ] + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "BASE": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port" + ] + }, + "HTTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "http_url", + "http_host", + "http_domain", + "http_request_line", + "http_response_line", + "http_request_header", + "http_response_header", + "http_request_content", + "http_response_content", + "http_request_body", + "http_response_body", + "http_request_body_key", + "http_response_body_key", + "http_proxy_flag", + "http_sequence", + "http_snapshot", + "http_cookie", + "http_referer", + "http_user_agent", + "http_content_length", + "http_content_type", + "http_set_cookie", + "http_version", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "http_url", + "common_server_port" + ] + }, + "MAIL": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "mail_protocol_type", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_subject_charset", + "mail_content", + "mail_content_charset", + "mail_attachment_name", + "mail_attachment_name_charset", + "mail_attachment_content", + "mail_eml_file", + "mail_snapshot" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "mail_from", + "mail_to", + "mail_subject" + ] + }, + "DNS": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_tc", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qdcount", + "dns_ancount", + "dns_nscount", + "dns_arcount", + "dns_qname", + "dns_qtype", + "dns_qclass", + "dns_cname", + "dns_sub", + "dns_rr" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_client_ip", + "dns_qr", + "dns_qname", + "dns_qtype" + ] + }, + "SSL": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_pinningst", + "ssl_intercept_state", + "ssl_server_side_latency", + "ssl_client_side_latency", + "ssl_server_side_version", + "ssl_client_side_version", + "ssl_cert_verify", + "ssl_error", + "ssl_con_latency_ms", + "ssl_ja3_fingerprint", + "ssl_ja3_hash", + "ssl_cert_issuer", + "ssl_cert_subject" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "ssl_sni", + "common_server_ip", + "common_server_port" + ] + }, + "QUIC": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "quic_version", + "quic_sni", + "quic_user_agent" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "quic_sni", + "common_server_ip", + "common_server_port" + ] + }, + "FTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "ftp_account", + "ftp_url", + "ftp_content", + "ftp_link_type" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "ftp_url", + "common_server_ip", + "common_server_port" + ] + }, + "BGP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "bgp_type", + "bgp_as_num", + "bgp_route" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "bgp_type", + "bgp_as_num", + "common_server_ip", + "common_server_port" + ] + }, + "SIP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration", + "sip_bye" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "common_server_ip", + "common_server_port" + ] + }, + "RTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "rtp_pcap_path" + ] + }, + "APP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "app_extra_info" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "common_app_id", + "common_app_label", + "app_extra_info", + "common_server_ip", + "common_server_port" + ] + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "common_schema_type" + ], + "tunnel_type": { + "GTP": [ + { + "name": "gtp_sgw_ip", + "label": "S-GW IP", + "type": "string" + }, + { + "name": "gtp_pgw_ip", + "label": "P-GW IP", + "type": "string" + }, + { + "name": "gtp_sgw_port", + "label": "S-GW Port", + "type": "int" + }, + { + "name": "gtp_pgw_port", + "label": "P-GW Port", + "type": "int" + }, + { + "name": "gtp_uplink_teid", + "label": "Uplink TEID", + "type": "long" + }, + { + "name": "gtp_downlink_teid", + "label": "Downlink TEID", + "type": "long" + } + ], + "MPLS": [ + { + "name": "mpls_c2s_direction_label", + "label": "Multiprotocol Label(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "mpls_s2c_direction_label", + "label": "Multiprotocol Label(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "VLAN": [ + { + "name": "vlan_c2s_direction_id", + "label": "VLAN Direction(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "vlan_s2c_direction_id", + "label": "VLAN Direction(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "ETHERNET": [ + { + "name": "source_mac", + "label": "Source MAC", + "type": "string" + }, + { + "name": "destination_mac", + "label": "Destination MAC", + "type": "string" + } + ], + "MULTIPATH_ETHERNET": [ + { + "name": "c2s_source_mac", + "label": "Source MAC(c2s)", + "type": "string" + }, + { + "name": "c2s_destination_mac", + "label": "Destination MAC(c2s)", + "type": "string" + }, + { + "name": "s2c_source_mac", + "label": "Source MAC(s2c)", + "type": "string" + }, + { + "name": "s2c_destination_mac", + "label": "Destination MAC(s2c)", + "type": "string" + } + ] + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn,radius_match", + "appendTo": "common_client_asn,common_subscriber_id" + } + }, + "type": "string" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "allow_query": "true", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "doc": { + "format": { + "functions": "set_value", + "param": "1" + } + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "MAIL", + "value": "MAIL" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "SSL", + "value": "SSL" + }, + { + "code": "QUIC", + "value": "QUIC" + }, + { + "code": "FTP", + "value": "FTP" + }, + { + "code": "SIP", + "value": "SIP" + }, + { + "code": "RTP", + "value": "RTP" + }, + { + "code": "APP", + "value": "APP" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "City A", + "value": "City A" + }, + { + "code": "City B", + "value": "City B" + }, + { + "code": "City C", + "value": "City C" + }, + { + "code": "City D", + "value": "City D" + }, + { + "code": "City E", + "value": "City E" + }, + { + "code": "City F", + "value": "City F" + }, + { + "code": "City G", + "value": "City G" + }, + { + "code": "City H", + "value": "City H" + }, + { + "code": "City I", + "value": "City I" + }, + { + "code": "City J", + "value": "City J" + }, + { + "code": "City K", + "value": "City K" + }, + { + "code": "City L", + "value": "City L" + }, + { + "code": "City M", + "value": "City M" + }, + { + "code": "City N", + "value": "City N" + }, + { + "code": "City O", + "value": "City O" + }, + { + "code": "City P", + "value": "City P" + }, + { + "code": "City Q", + "value": "City Q" + }, + { + "code": "City R", + "value": "City R" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Ethernet" + }, + { + "code": "8", + "value": "PPP" + }, + { + "code": "12", + "value": "CiscoHDLC" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "type": "string", + "doc": { + "allow_query": "true" + } + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string" + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string" + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "allow_query": "true", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "allow_query": "true", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ] + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "http_url", + "label": "HTTP.URL", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_host", + "label": "HTTP.Host", + "doc": { + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "http_domain", + "label": "HTTP.Domain", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "http_request_line", + "label": "HTTP.Request Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_line", + "label": "HTTP.Response Line", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_request_header", + "label": "HTTP.Request Header", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_response_header", + "label": "HTTP.Response Header", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_request_content", + "label": "HTTP.Request Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_response_content", + "label": "HTTP.Response Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_request_body", + "label": "HTTP.Request Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_response_body", + "label": "HTTP.Response Body", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "http_request_body_key", + "label": "HTTP.Request Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_response_body_key", + "label": "HTTP.Response Body Key", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "http_proxy_flag", + "label": "HTTP.Proxy Flag", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_sequence", + "label": "HTTP.Sequence", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "http_snapshot", + "label": "HTTP.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "http_cookie", + "label": "HTTP.Cookie", + "type": "string" + }, + { + "name": "http_referer", + "label": "HTTP.Referer", + "type": "string" + }, + { + "name": "http_user_agent", + "label": "HTTP.User Agent", + "type": "string" + }, + { + "name": "http_content_length", + "label": "HTTP.Content Length", + "type": "string" + }, + { + "name": "http_content_type", + "label": "HTTP.Content Type", + "type": "string" + }, + { + "name": "http_set_cookie", + "label": "HTTP.Set Cookie", + "type": "string" + }, + { + "name": "http_version", + "label": "HTTP.Version", + "type": "string" + }, + { + "name": "http_response_latency_ms", + "label": "HTTP.Response Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "http_action_file_size", + "label": "HTTP.Action File Size", + "type": "int" + }, + { + "name": "http_session_duration_ms", + "label": "HTTP.Session Duration(ms)", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "mail_protocol_type", + "label": "Mail.Protocol Type", + "type": "string" + }, + { + "name": "mail_account", + "label": "Mail.Account", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "mail_from_cmd", + "label": "Mail.From CMD", + "type": "string" + }, + { + "name": "mail_to_cmd", + "label": "Mail.To CMD", + "type": "string" + }, + { + "name": "mail_from", + "label": "Mail.From", + "doc": { + "allow_query": "true", + "constraints": { + "type": "email" + } + }, + "type": "string" + }, + { + "name": "mail_to", + "label": "Mail.To", + "doc": { + "allow_query": "true", + "constraints": { + "type": "email" + } + }, + "type": "string" + }, + { + "name": "mail_cc", + "label": "Mail.CC", + "type": "string" + }, + { + "name": "mail_bcc", + "label": "Mail.BCC", + "type": "string" + }, + { + "name": "mail_subject", + "label": "Mail.Subject", + "doc": { + "allow_query": "true", + "format": { + "functions": "decode_of_base64", + "param": "$.mail_subject_charset" + } + }, + "type": "string" + }, + { + "name": "mail_subject_charset", + "label": "Mail.Subject Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_content", + "label": "Mail.Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_content_charset", + "label": "Mail.Content Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_attachment_name", + "label": "Mail.Attachment", + "doc": { + "format": { + "functions": "decode_of_base64", + "param": "$.mail_attachment_name_charset" + } + }, + "type": "string" + }, + { + "name": "mail_attachment_name_charset", + "label": "Mail.Attachment Charset", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_attachment_content", + "label": "Mail.Attachment Content", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_eml_file", + "label": "Mail.EML File", + "doc": { + "constraints": { + "type": "file" + }, + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "mail_snapshot", + "label": "Mail.Snapshot", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "dns_message_id", + "label": "DNS.Message ID", + "type": "int" + }, + { + "name": "dns_qr", + "label": "DNS.QR", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "RESPONSE" + } + ] + }, + "type": "int" + }, + { + "name": "dns_opcode", + "label": "DNS.OPCODE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "IQUERY" + }, + { + "code": "2", + "value": "STATUS" + }, + { + "code": "5", + "value": "UPDATE" + } + ] + }, + "type": "int" + }, + { + "name": "dns_aa", + "label": "DNS.AA", + "type": "int" + }, + { + "name": "dns_tc", + "label": "DNS.TC", + "type": "int" + }, + { + "name": "dns_rd", + "label": "DNS.RD", + "type": "int" + }, + { + "name": "dns_ra", + "label": "DNS.RA", + "type": "int" + }, + { + "name": "dns_rcode", + "label": "DNS.RCODE", + "type": "int" + }, + { + "name": "dns_qdcount", + "label": "DNS.QDCOUNT", + "type": "int" + }, + { + "name": "dns_ancount", + "label": "DNS.ANCOUNT", + "type": "int" + }, + { + "name": "dns_nscount", + "label": "DNS.NSCOUNT", + "type": "int" + }, + { + "name": "dns_arcount", + "label": "DNS.ARCOUNT", + "type": "int" + }, + { + "name": "dns_qname", + "label": "DNS.QNAME", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "dns_qtype", + "label": "DNS.QTYPE", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "A" + }, + { + "code": "2", + "value": "NS" + }, + { + "code": "5", + "value": "CNAME" + }, + { + "code": "6", + "value": "SOA" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "12", + "value": "PTR" + }, + { + "code": "13", + "value": "HINFO" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "15", + "value": "MX" + }, + { + "code": "28", + "value": "AAAA" + } + ] + }, + "type": "int" + }, + { + "name": "dns_qclass", + "label": "DNS.QCLASS", + "type": "int" + }, + { + "name": "dns_cname", + "label": "DNS.CNAME", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "dns_sub", + "label": "DNS.SUB", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "DNS" + }, + { + "code": "2", + "value": "DNSSEC" + } + ] + }, + "type": "int" + }, + { + "name": "dns_rr", + "label": "DNS.RR", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_version", + "label": "SSL.Version", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "ssl_sni", + "label": "SSL.SNI", + "doc": { + "allow_query": "true", + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "ssl_san", + "label": "SSL.SAN", + "type": "string" + }, + { + "name": "ssl_cn", + "label": "SSL.CN", + "type": "string" + }, + { + "name": "ssl_pinningst", + "label": "SSL.Pinning", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Not Pinning" + }, + { + "code": "1", + "value": "Pinning" + }, + { + "code": "2", + "value": "Maybe Pinning" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_intercept_state", + "label": "SSL.Intercept State", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Passthrough" + }, + { + "code": "1", + "value": "Intercept" + }, + { + "code": "2", + "value": "Shutdown" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_server_side_latency", + "label": "SSL.Server Side Latency(ms)", + "type": "int" + }, + { + "name": "ssl_client_side_latency", + "label": "SSL.Client Side Latency(ms)", + "type": "int" + }, + { + "name": "ssl_server_side_version", + "label": "SSL.Server Side Version", + "type": "string" + }, + { + "name": "ssl_client_side_version", + "label": "SSL.Client Side Version", + "type": "string" + }, + { + "name": "ssl_cert_verify", + "label": "SSL.Certificate Verify", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ] + }, + "type": "int" + }, + { + "name": "ssl_error", + "label": "SSL.Error", + "type": "string" + }, + { + "name": "ssl_con_latency_ms", + "label": "SSL.Connection Latency(ms)", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "ssl_ja3_fingerprint", + "label": "SSL.JA3", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "ssl_ja3_hash", + "label": "SSL.JA3 hash", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ssl_cert_issuer", + "label": "SSL.Issuer", + "doc": { + "constraints": { + "type": "items" + } + }, + "type": "string" + }, + { + "name": "ssl_cert_subject", + "label": "SSL.Subject", + "doc": { + "constraints": { + "type": "items" + } + }, + "type": "string" + }, + { + "name": "quic_version", + "label": "Quic.Version", + "type": "string" + }, + { + "name": "quic_sni", + "label": "Quic.SNI", + "doc": { + "allow_query": "true", + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "type": "string" + }, + { + "name": "quic_user_agent", + "label": "Quic.User Agent", + "type": "string" + }, + { + "name": "ftp_account", + "label": "FTP.Account", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "ftp_url", + "label": "FTP.URL", + "type": "string" + }, + { + "name": "ftp_content", + "label": "FTP.Content", + "type": "string" + }, + { + "name": "ftp_link_type", + "label": "FTP.Link Type", + "type": "string" + }, + { + "name": "bgp_type", + "label": "BGP.Type", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "bgp_as_num", + "label": "BGP.AS Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "bgp_route", + "label": "BGP.Route", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_calling_account", + "label": "VoIP.Calling Account", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_called_account", + "label": "VoIP.Called Account", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_calling_number", + "label": "VoIP.Calling Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "voip_called_number", + "label": "VoIP.Called Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "streaming_media_url", + "label": "Streaming.Media URL", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "streaming_media_protocol", + "label": "Streaming.Media Protocol", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "app_extra_info", + "label": "APP.Extra Info", + "type": "string" + }, + { + "name": "sip_call_id", + "label": "SIP.Call-ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "sip_originator_description", + "label": "SIP.Originator", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "sip_responder_description", + "label": "SIP.Responder", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "sip_user_agent", + "label": "SIP.User-Agent", + "type": "string" + }, + { + "name": "sip_server", + "label": "SIP.Server", + "type": "string" + }, + { + "name": "sip_originator_sdp_connect_ip", + "label": "SIP.Originator IP", + "type": "string" + }, + { + "name": "sip_originator_sdp_media_port", + "label": "SIP.Originator Port", + "type": "int" + }, + { + "name": "sip_originator_sdp_media_type", + "label": "SIP.Originator Media Type", + "type": "string" + }, + { + "name": "sip_originator_sdp_content", + "label": "SIP.Originator Content", + "type": "string" + }, + { + "name": "sip_responder_sdp_connect_ip", + "label": "SIP.Responder IP", + "type": "string" + }, + { + "name": "sip_responder_sdp_media_port", + "label": "SIP.Responder Port", + "type": "int" + }, + { + "name": "sip_responder_sdp_media_type", + "label": "SIP.Responder Media Type", + "type": "string" + }, + { + "name": "sip_responder_sdp_content", + "label": "SIP.Responder Content", + "type": "string" + }, + { + "name": "sip_duration", + "label": "SIP.Duration", + "type": "int" + }, + { + "name": "sip_bye", + "label": "SIP.Bye", + "type": "string" + }, + { + "name": "rtp_payload_type_c2s", + "label": "RTP.Payload Type(c2s)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_payload_type_s2c", + "label": "RTP.Payload Type(s2c)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_pcap_path", + "label": "RTP.PCAP", + "doc": { + "constraints": { + "type": "file" + } + }, + "type": "string" + }, + { + "name": "rtp_originator_dir", + "label": "RTP.Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "unknown" + }, + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + } + ], + "visibility": "hidden" + }, + "type": "int" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/sys_packet_capture_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/sys_packet_capture_log.json new file mode 100644 index 0000000..73e9454 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/sys_packet_capture_log.json @@ -0,0 +1,750 @@ +{ + "type": "record", + "name": "sys_packet_capture_log", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time" + }, + "fields": [ + { + "name": "common_recv_time", + "type": "long", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "label": "Receive Time" + }, + { + "name": "common_log_id", + "type": "long", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "label": "Log ID" + }, + { + "name": "common_policy_id", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Policy ID" + }, + { + "name": "common_subscriber_id", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "Subscriber ID" + }, + { + "name": "common_imei", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "IMEI" + }, + { + "name": "common_imsi", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "IMSI" + }, + { + "name": "common_phone_number", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "Phone Number" + }, + { + "name": "common_client_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "Client IP" + }, + { + "name": "common_internal_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "Internal IP" + }, + { + "name": "common_client_port", + "type": "int", + "label": "Client Port" + }, + { + "name": "common_l4_protocol", + "type": "string", + "label": "L4 Protocol" + }, + { + "name": "common_address_type", + "type": "int", + "doc": { + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "label": "Address Type" + }, + { + "name": "common_server_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "Server IP" + }, + { + "name": "common_server_port", + "type": "int", + "doc": { + "allow_query": "true" + }, + "label": "Server Port" + }, + { + "name": "common_external_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "External IP" + }, + { + "name": "common_action", + "type": "int", + "doc": { + "allow_query": "true", + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "label": "Action" + }, + { + "name": "common_direction", + "type": "int", + "doc": { + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "label": "Direction" + }, + { + "name": "common_entrance_id", + "type": "int", + "doc": { + "visibility": "disabled" + }, + "label": "Entrance ID" + }, + { + "name": "common_sled_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "Sled IP" + }, + { + "name": "common_client_location", + "type": "string", + "label": "Client Location" + }, + { + "name": "common_client_asn", + "type": "string", + "label": "Client ASN" + }, + { + "name": "common_server_location", + "type": "string", + "label": "Server Location" + }, + { + "name": "common_server_asn", + "type": "string", + "label": "Server ASN" + }, + { + "name": "common_sessions", + "type": "long", + "label": "Sessions" + }, + { + "name": "common_c2s_pkt_num", + "type": "long", + "label": "Packets Sent" + }, + { + "name": "common_s2c_pkt_num", + "type": "long", + "label": "Packets Received" + }, + { + "name": "common_c2s_byte_num", + "type": "long", + "label": "Bytes Sent" + }, + { + "name": "common_s2c_byte_num", + "type": "long", + "label": "Bytes Received" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "type": "long" + }, + { + "name": "common_service", + "type": "int", + "doc": { + "visibility": "disabled" + }, + "label": "Service" + }, + { + "name": "common_schema_type", + "type": "string", + "doc": { + "data": [ + { + "code": "BASE", + "value": "BASE" + }, + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "MAIL", + "value": "MAIL" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "SSL", + "value": "SSL" + }, + { + "code": "FTP", + "value": "FTP" + } + ], + "visibility": "hidden" + }, + "label": "Schema Type" + }, + { + "name": "common_user_tags", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "User Tags" + }, + { + "name": "common_sub_action", + "type": "string", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "label": "Sub Action" + }, + { + "name": "common_user_region", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "User Region" + }, + { + "name": "common_device_id", + "type": "string", + "label": "Device ID" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "ISP" + }, + { + "name": "common_device_tag", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Device Tag" + }, + { + "name": "common_data_center", + "type": "string", + "label": "Data Center" + }, + { + "name": "common_encapsulation", + "type": "int", + "doc": { + "data": [ + { + "code": "0", + "value": "Ethernet" + }, + { + "code": "8", + "value": "PPP" + }, + { + "code": "12", + "value": "CiscoHDLC" + } + ] + }, + "label": "Encapsulation" + }, + { + "name": "common_app_label", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "Application Label" + }, + { + "name": "common_tunnels", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Tunnels" + }, + { + "name": "common_protocol_label", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Protocol Label" + }, + { + "name": "common_app_id", + "type": "string", + "label": "Application ID", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "type": "string", + "label": "Surrogate ID", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "type": "string", + "label": "L7 Protocol" + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + }, + "visibility": "hidden" + }, + "label": "Start Time" + }, + { + "name": "common_end_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + }, + "visibility": "hidden" + }, + "label": "End Time" + }, + { + "name": "common_establish_latency_ms", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Establish Latency(ms)" + }, + { + "name": "common_con_duration_ms", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Duration(ms)" + }, + { + "name": "common_stream_dir", + "type": "int", + "doc": { + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ] + }, + "label": "Stream Direction" + }, + { + "name": "common_address_list", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "Address List" + }, + { + "name": "common_has_dup_traffic", + "type": "int", + "doc": { + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ], + "visibility": "hidden" + }, + "label": "Duplication Traffic" + }, + { + "name": "common_stream_error", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Stream Error" + }, + { + "name": "common_stream_trace_id", + "type": "long", + "doc": { + "allow_query": "true" + }, + "label": "Session ID" + }, + { + "name": "common_link_info_c2s", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Link Info(c2s)" + }, + { + "name": "common_link_info_s2c", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Link Info(s2c)" + }, + { + "name": "common_c2s_ipfrag_num", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Fragmentation Packets(c2s)" + }, + { + "name": "common_s2c_ipfrag_num", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Fragmentation Packets(s2c)" + }, + { + "name": "common_c2s_tcp_lostlen", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Sequence Gap Loss(c2s)" + }, + { + "name": "common_s2c_tcp_lostlen", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Sequence Gap Loss(s2c)" + }, + { + "name": "common_c2s_tcp_unorder_num", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Unorder Packets(c2s)" + }, + { + "name": "common_s2c_tcp_unorder_num", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Unorder Packets(s2c)" + }, + { + "name": "common_c2s_pkt_retrans", + "type": "long", + "label": "Packet Retransmission(c2s)" + }, + { + "name": "common_s2c_pkt_retrans", + "type": "long", + "label": "Packet Retransmission(s2c)" + }, + { + "name": "common_c2s_byte_retrans", + "type": "long", + "label": "Byte Retransmission(c2s)" + }, + { + "name": "common_s2c_byte_retrans", + "type": "long", + "label": "Byte Retransmission(s2c)" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "type": "int", + "doc": { + "visibility": "hidden" + }, + "label": "First TTL" + }, + { + "name": "common_processing_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "label": "Processing Time" + }, + { + "name": "nic_name", + "type": "string", + "label": "Nic Name" + }, + { + "name": "origin_source_mac", + "type": "string", + "label": "Origin Source Mac" + }, + { + "name": "origin_dest_mac", + "type": "string", + "label": "Origin Dest Mac" + }, + { + "name": "packet_url", + "type": "string", + "label": "Packet URL" + }, + { + "name": "pcap_storage_task_id", + "type": "int", + "label": "Task ID" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/sys_storage_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/sys_storage_log.json new file mode 100644 index 0000000..3bb3224 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/sys_storage_log.json @@ -0,0 +1,38 @@ +{ + "type": "record", + "name": "sys_storage_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "log_type", + "type": "string" + }, + { + "name": "data_center", + "type": "string" + }, + { + "name": "max_size", + "type": "long" + }, + { + "name": "used_size", + "type": "long" + }, + { + "name": "aggregate_size", + "type": "long" + }, + { + "name": "last_storage", + "type": "long" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/tables_cluster.json b/parcels/roles/services/galaxy-qgw-service/files/config/tables_cluster.json new file mode 100644 index 0000000..4765d85 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/tables_cluster.json @@ -0,0 +1,11 @@ +{ + "namespace": "system", + "type": "record", + "name": "tables_cluster", + "fields": [ + { + "name": "database", + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/top_client_ip_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/top_client_ip_log.json new file mode 100644 index 0000000..78f3867 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/top_client_ip_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_client_ip_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "source", + "type": "string" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/top_external_host_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/top_external_host_log.json new file mode 100644 index 0000000..68c229e --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/top_external_host_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_external_host_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "destination", + "type": "string" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/top_internal_host_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/top_internal_host_log.json new file mode 100644 index 0000000..75347a5 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/top_internal_host_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_internal_host_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "source", + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/top_server_ip_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/top_server_ip_log.json new file mode 100644 index 0000000..74258f1 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/top_server_ip_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_server_ip_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "destination", + "type": "string" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/top_urls_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/top_urls_log.json new file mode 100644 index 0000000..7a0cc9b --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/top_urls_log.json @@ -0,0 +1,22 @@ +{ + "type": "record", + "name": "top_urls_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "url", + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/top_user_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/top_user_log.json new file mode 100644 index 0000000..ebddb24 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/top_user_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_user_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "subscriber_id", + "type": "string" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/top_website_domain_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/top_website_domain_log.json new file mode 100644 index 0000000..df86ea9 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/top_website_domain_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "top_website_domain_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "domain", + "type": "string" + }, + { + "name": "order_by", + "type": "string" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "session_num", + "type": "long" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/traffic_app_stat_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/traffic_app_stat_log.json new file mode 100644 index 0000000..083ef7b --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/traffic_app_stat_log.json @@ -0,0 +1,42 @@ +{ + "type": "record", + "name": "traffic_app_stat_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "app_name", + "type": "string" + }, + { + "name": "sub_app_name", + "type": "string" + }, + { + "name": "session_num", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/traffic_metrics_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/traffic_metrics_log.json new file mode 100644 index 0000000..6e0a690 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/traffic_metrics_log.json @@ -0,0 +1,214 @@ +{ + "type": "record", + "name": "traffic_metrics_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "allow_conn_num", + "type": "long" + }, + { + "name": "allow_in_bytes", + "type": "long" + }, + { + "name": "allow_in_packets", + "type": "long" + }, + { + "name": "allow_out_bytes", + "type": "long" + }, + { + "name": "allow_out_packets", + "type": "long" + }, + { + "name": "close_conn_num", + "type": "long" + }, + { + "name": "default_conn_num", + "type": "long" + }, + { + "name": "default_in_bytes", + "type": "long" + }, + { + "name": "default_in_packets", + "type": "long" + }, + { + "name": "default_out_bytes", + "type": "long" + }, + { + "name": "default_out_packets", + "type": "long" + }, + { + "name": "deny_conn_num", + "type": "long" + }, + { + "name": "deny_in_bytes", + "type": "long" + }, + { + "name": "deny_in_packets", + "type": "long" + }, + { + "name": "deny_out_bytes", + "type": "long" + }, + { + "name": "deny_out_packets", + "type": "long" + }, + { + "name": "device_id", + "type": "string" + }, + { + "name": "entrance_id", + "type": "long" + }, + { + "name": "intercept_conn_num", + "type": "long" + }, + { + "name": "intercept_in_bytes", + "type": "long" + }, + { + "name": "intercept_in_packets", + "type": "long" + }, + { + "name": "intercept_out_bytes", + "type": "long" + }, + { + "name": "intercept_out_packets", + "type": "long" + }, + { + "name": "established_conn_num", + "type": "long" + }, + { + "name": "monitor_conn_num", + "type": "long" + }, + { + "name": "monitor_in_bytes", + "type": "long" + }, + { + "name": "monitor_in_packets", + "type": "long" + }, + { + "name": "monitor_out_bytes", + "type": "long" + }, + { + "name": "monitor_out_packets", + "type": "long" + }, + { + "name": "new_conn_num", + "type": "long" + }, + { + "name": "total_in_bytes", + "type": "long" + }, + { + "name": "total_in_packets", + "type": "long" + }, + { + "name": "total_out_bytes", + "type": "long" + }, + { + "name": "total_out_packets", + "type": "long" + }, + { + "name": "alert_bytes", + "type": "long" + }, + { + "name": "hijk_bytes", + "type": "long" + }, + { + "name": "ins_bytes", + "type": "long" + }, + { + "name": "intcp_allow_num", + "type": "long" + }, + { + "name": "intcp_deny_num", + "type": "long" + }, + { + "name": "intcp_hijk_num", + "type": "long" + }, + { + "name": "intcp_ins_num", + "type": "long" + }, + { + "name": "intcp_mon_num", + "type": "long" + }, + { + "name": "intcp_rdirt_num", + "type": "long" + }, + { + "name": "intcp_repl_num", + "type": "long" + }, + { + "name": "maybe_pinning_num", + "type": "long" + }, + { + "name": "not_pinning_num", + "type": "long" + }, + { + "name": "pinning_num", + "type": "long" + }, + { + "name": "ad_cc_bytes", + "type": "long" + }, + { + "name": "ad_flood_bytes", + "type": "long" + }, + { + "name": "ad_reflection_bytes", + "type": "long" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/traffic_protocol_stat_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/traffic_protocol_stat_log.json new file mode 100644 index 0000000..0b7df7b --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/traffic_protocol_stat_log.json @@ -0,0 +1,74 @@ +{ + "type": "record", + "name": "traffic_protocol_stat_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "protocol_id", + "type": "string" + }, + { + "name": "isp", + "type": "string" + }, + { + "name": "entrance_id", + "type": "long" + }, + { + "name": "data_center", + "type": "string" + }, + { + "name": "sessions", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "c2s_ipfrag_num", + "type": "long" + }, + { + "name": "s2c_ipfrag_num", + "type": "long" + }, + { + "name": "c2s_tcp_lostlen", + "type": "long" + }, + { + "name": "s2c_tcp_lostlen", + "type": "long" + }, + { + "name": "c2s_tcp_unorder_num", + "type": "long" + }, + { + "name": "s2c_tcp_unorder_num", + "type": "long" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/traffic_summary_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/traffic_summary_log.json new file mode 100644 index 0000000..10bf556 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/traffic_summary_log.json @@ -0,0 +1,74 @@ +{ + "type": "record", + "name": "traffic_summary_log", + "namespace": "druid", + "doc": { + "partition_key": "__time" + }, + "fields": [ + { + "name": "__time", + "type": "long" + }, + { + "name": "isp", + "type": "string" + }, + { + "name": "entrance_id", + "type": "long" + }, + { + "name": "data_center", + "type": "string" + }, + { + "name": "schema_type", + "type": "string" + }, + { + "name": "ip_object", + "type": "string" + }, + { + "name": "sessions", + "type": "long" + }, + { + "name": "c2s_pkt_num", + "type": "long" + }, + { + "name": "s2c_pkt_num", + "type": "long" + }, + { + "name": "c2s_byte_num", + "type": "long" + }, + { + "name": "s2c_byte_num", + "type": "long" + }, + { + "name": "one_sided_connections", + "type": "long" + }, + { + "name": "uncategorized_bytes", + "type": "long" + }, + { + "name": "fragmentation_packets", + "type": "long" + }, + { + "name": "sequence_gap_loss", + "type": "long" + }, + { + "name": "unorder_packets", + "type": "long" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/transaction_record_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/transaction_record_log.json new file mode 100644 index 0000000..5657c8b --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/transaction_record_log.json @@ -0,0 +1,2795 @@ +{ + "type": "record", + "name": "transaction_record_log", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_stream_trace_id", + "partition_key": "common_recv_time", + "functions": { + "aggregation": [ + { + "name": "COUNT", + "label": "COUNT", + "function": "count(expr)" + }, + { + "name": "COUNT_DISTINCT", + "label": "COUNT_DISTINCT", + "function": "count(distinct expr)" + }, + { + "name": "AVG", + "label": "AVG", + "function": "avg(expr)" + }, + { + "name": "SUM", + "label": "SUM", + "function": "sum(expr)" + }, + { + "name": "MAX", + "label": "MAX", + "function": "max(expr)" + }, + { + "name": "MIN", + "label": "MIN", + "function": "min(expr)" + } + ], + "operator": [ + { + "name": "=", + "label": "=", + "function": "expr = value" + }, + { + "name": "!=", + "label": "!=", + "function": "expr != value" + }, + { + "name": ">", + "label": ">", + "function": "expr > value" + }, + { + "name": "<", + "label": "<", + "function": "expr < value" + }, + { + "name": ">=", + "label": ">=", + "function": "expr >= value" + }, + { + "name": "<=", + "label": "<=", + "function": "expr <= value" + }, + { + "name": "has", + "label": "HAS", + "function": "has(expr, value)" + }, + { + "name": "in", + "label": "IN", + "function": "expr in (values)" + }, + { + "name": "not in", + "label": "NOT IN", + "function": "expr not in (values)" + }, + { + "name": "like", + "label": "LIKE", + "function": "expr like value" + }, + { + "name": "not like", + "label": "NOT LIKE", + "function": "expr not like value" + }, + { + "name": "notEmpty", + "label": "NOT EMPTY", + "function": "notEmpty(expr)" + }, + { + "name": "empty", + "label": "EMPTY", + "function": "empty(expr)" + } + ] + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_l4_protocol", + "common_l7_protocol", + "common_data_center", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_subscriber_id", + "common_sled_ip", + "common_device_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_sessions", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url" + ], + "filters": [ + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_l4_protocol", + "common_l7_protocol", + "common_stream_dir", + "common_direction", + "common_data_center", + "common_sled_ip", + "common_device_id", + "common_schema_type", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_imei", + "common_imsi", + "common_phone_number", + "http_host", + "http_domain", + "http_url" + ], + "references": { + "aggregation": [ + { + "type": "int", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "long", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "float", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "double", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "string", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "date", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + } + ], + "operator": [ + { + "type": "int", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "long", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "float", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "double", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "string", + "functions": "=,!=,in,not in,like,not like,notEmpty,empty" + }, + { + "type": "date", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "timestamp", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type":"array", + "functions":"has" + } + ] + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "BASE": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_service_category", + "common_l7_protocol", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port" + ] + }, + "HTTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_service_category", + "common_l7_protocol", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "http_url", + "http_host", + "http_domain", + "http_request_line", + "http_response_line", + "http_request_header", + "http_response_header", + "http_request_content", + "http_response_content", + "http_request_body", + "http_response_body", + "http_request_body_key", + "http_response_body_key", + "http_proxy_flag", + "http_sequence", + "http_snapshot", + "http_cookie", + "http_referer", + "http_user_agent", + "http_content_length", + "http_content_type", + "http_set_cookie", + "http_version", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "http_url", + "common_server_port" + ] + }, + "MAIL": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip" + ] + }, + "DNS": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_tc", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qdcount", + "dns_ancount", + "dns_nscount", + "dns_arcount", + "dns_qname", + "dns_qtype", + "dns_qclass", + "dns_cname", + "dns_sub", + "dns_rr" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_client_ip", + "dns_qr", + "dns_qname", + "dns_qtype" + ] + }, + "SSL": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port" + ] + }, + "QUIC": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port" + ] + }, + "FTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port" + ] + }, + "BGP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port" + ] + }, + "SIP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration", + "sip_bye" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "common_server_ip", + "common_server_port" + ] + }, + "RTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port" + ] + }, + "APP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_app_id", + "common_app_label", + "common_server_ip", + "common_server_port" + ] + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "common_schema_type" + ], + "tunnel_type": { + "GTP": [ + { + "name": "gtp_sgw_ip", + "label": "S-GW IP", + "type": "string" + }, + { + "name": "gtp_pgw_ip", + "label": "P-GW IP", + "type": "string" + }, + { + "name": "gtp_sgw_port", + "label": "S-GW Port", + "type": "int" + }, + { + "name": "gtp_pgw_port", + "label": "P-GW Port", + "type": "int" + }, + { + "name": "gtp_uplink_teid", + "label": "Uplink TEID", + "type": "long" + }, + { + "name": "gtp_downlink_teid", + "label": "Downlink TEID", + "type": "long" + } + ], + "MPLS": [ + { + "name": "mpls_c2s_direction_label", + "label": "Multiprotocol Label(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "mpls_s2c_direction_label", + "label": "Multiprotocol Label(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "VLAN": [ + { + "name": "vlan_c2s_direction_id", + "label": "VLAN Direction(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "vlan_s2c_direction_id", + "label": "VLAN Direction(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "ETHERNET": [ + { + "name": "source_mac", + "label": "Source MAC", + "type": "string" + }, + { + "name": "destination_mac", + "label": "Destination MAC", + "type": "string" + } + ], + "MULTIPATH_ETHERNET": [ + { + "name": "c2s_source_mac", + "label": "Source MAC(c2s)", + "type": "string" + }, + { + "name": "c2s_destination_mac", + "label": "Destination MAC(c2s)", + "type": "string" + }, + { + "name": "s2c_source_mac", + "label": "Source MAC(s2c)", + "type": "string" + }, + { + "name": "s2c_destination_mac", + "label": "Destination MAC(s2c)", + "type": "string" + } + ] + } + }, + "fields": [ + { + "name": "common_recv_time", + "type": "long", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "label": "Receive Time" + }, + { + "name": "common_log_id", + "type": "long", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "label": "Log ID" + }, + { + "name": "common_policy_id", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Policy ID" + }, + { + "name": "common_subscriber_id", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "Subscriber ID" + }, + { + "name": "common_imei", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "IMEI" + }, + { + "name": "common_imsi", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "IMSI" + }, + { + "name": "common_phone_number", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "Phone Number" + }, + { + "name": "common_client_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn,radius_match", + "appendTo": "common_client_asn,common_subscriber_id" + } + }, + "label": "Client IP" + }, + { + "name": "common_internal_ip", + "type": "string", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "label": "Internal IP" + }, + { + "name": "common_client_port", + "type": "int", + "doc": { + "allow_query": "true" + }, + "label": "Client Port" + }, + { + "name": "common_l4_protocol", + "type": "string", + "label": "L4 Protocol" + }, + { + "name": "common_address_type", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "label": "Address Type" + }, + { + "name": "common_server_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "label": "Server IP" + }, + { + "name": "common_server_port", + "type": "int", + "doc": { + "allow_query": "true" + }, + "label": "Server Port" + }, + { + "name": "common_external_ip", + "type": "string", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "label": "External IP" + }, + { + "name": "common_action", + "type": "int", + "doc": { + "visibility": "hidden", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "label": "Action" + }, + { + "name": "common_direction", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "label": "Direction" + }, + { + "name": "common_entrance_id", + "type": "int", + "doc": { + "visibility": "disabled" + }, + "label": "Entrance ID" + }, + { + "name": "common_sled_ip", + "type": "string", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "label": "Sled IP" + }, + { + "name": "common_client_location", + "type": "string", + "label": "Client Location" + }, + { + "name": "common_client_asn", + "type": "string", + "label": "Client ASN" + }, + { + "name": "common_server_location", + "type": "string", + "label": "Server Location" + }, + { + "name": "common_server_asn", + "type": "string", + "label": "Server ASN" + }, + { + "name": "common_sessions", + "type": "long", + "label": "Sessions" + }, + { + "name": "common_c2s_pkt_num", + "type": "long", + "label": "Packets Sent" + }, + { + "name": "common_s2c_pkt_num", + "type": "long", + "label": "Packets Received" + }, + { + "name": "common_c2s_byte_num", + "type": "long", + "label": "Bytes Sent" + }, + { + "name": "common_s2c_byte_num", + "type": "long", + "label": "Bytes Received" + }, + { + "name": "common_c2s_pkt_diff", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Packets Sent(Diff)" + }, + { + "name": "common_s2c_pkt_diff", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Packets Received(Diff)" + }, + { + "name": "common_c2s_byte_diff", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Bytes Sent(Diff)" + }, + { + "name": "common_s2c_byte_diff", + "type": "long", + "doc": { + "visibility": "hidden" + }, + "label": "Bytes Received(Diff)" + }, + { + "name": "common_service", + "type": "int", + "doc": { + "visibility": "disabled" + }, + "label": "Service" + }, + { + "name": "common_schema_type", + "type": "string", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "BASE", + "value": "BASE" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "SIP", + "value": "SIP" + } + ], + "allow_query": "true" + }, + "label": "Schema Type" + }, + { + "name": "common_user_tags", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "User Tags" + }, + { + "name": "common_sub_action", + "type": "string", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "label": "Sub Action" + }, + { + "name": "common_user_region", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "User Region" + }, + { + "name": "common_device_id", + "type": "string", + "label": "Device ID" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "ISP" + }, + { + "name": "common_device_tag", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Device Tag" + }, + { + "name": "common_data_center", + "type": "string", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "City A", + "value": "City A" + }, + { + "code": "City B", + "value": "City B" + }, + { + "code": "City C", + "value": "City C" + }, + { + "code": "City D", + "value": "City D" + }, + { + "code": "City E", + "value": "City E" + }, + { + "code": "City F", + "value": "City F" + }, + { + "code": "City G", + "value": "City G" + }, + { + "code": "City H", + "value": "City H" + }, + { + "code": "City I", + "value": "City I" + }, + { + "code": "City J", + "value": "City J" + }, + { + "code": "City K", + "value": "City K" + }, + { + "code": "City L", + "value": "City L" + }, + { + "code": "City M", + "value": "City M" + }, + { + "code": "City N", + "value": "City N" + }, + { + "code": "City O", + "value": "City O" + }, + { + "code": "City P", + "value": "City P" + }, + { + "code": "City Q", + "value": "City Q" + }, + { + "code": "City R", + "value": "City R" + } + ], + "allow_query": "true" + }, + "label": "Data Center" + }, + { + "name": "common_encapsulation", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Ethernet" + }, + { + "code": "8", + "value": "PPP" + }, + { + "code": "12", + "value": "CiscoHDLC" + } + ], + "visibility": "hidden" + }, + "label": "Encapsulation" + }, + { + "name": "common_app_label", + "type": "string", + "label": "Application Label" + }, + { + "name": "common_tunnels", + "type": "string", + "label": "Tunnels" + }, + { + "name": "common_protocol_label", + "type": "string", + "label": "Protocol Label" + }, + { + "name": "common_app_id", + "type": "string", + "label": "Application ID", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "type": "string", + "label": "Surrogate ID", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "type": "string", + "label": "L7 Protocol" + }, + { + "name": "common_service_category", + "type": { + "type": "array", + "items": "int" + }, + "doc": { + "constraints": { + "operator_functions": "has" + }, + "allow_query": "true", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "label": "FQDN Category" + }, + { + "name": "common_start_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "label": "Start Time" + }, + { + "name": "common_end_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "label": "End Time" + }, + { + "name": "common_establish_latency_ms", + "type": "long", + "label": "Establish Latency(ms)" + }, + { + "name": "common_con_duration_ms", + "type": "long", + "label": "Duration(ms)" + }, + { + "name": "common_stream_dir", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ], + "allow_query": "true" + }, + "label": "Stream Direction" + }, + { + "name": "common_address_list", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "Address List" + }, + { + "name": "common_has_dup_traffic", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ], + "visibility": "hidden" + }, + "label": "Duplication Traffic" + }, + { + "name": "common_stream_error", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Stream Error" + }, + { + "name": "common_stream_trace_id", + "type": "long", + "doc": { + "allow_query": "true" + }, + "label": "Session ID" + }, + { + "name": "common_link_info_c2s", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Link Info(c2s)" + }, + { + "name": "common_link_info_s2c", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "Link Info(s2c)" + }, + { + "name": "common_c2s_ipfrag_num", + "type": "long", + "label": "Fragmentation Packets(c2s)" + }, + { + "name": "common_s2c_ipfrag_num", + "type": "long", + "label": "Fragmentation Packets(s2c)" + }, + { + "name": "common_c2s_tcp_lostlen", + "type": "long", + "label": "Sequence Gap Loss(c2s)" + }, + { + "name": "common_s2c_tcp_lostlen", + "type": "long", + "label": "Sequence Gap Loss(s2c)" + }, + { + "name": "common_c2s_tcp_unorder_num", + "type": "long", + "label": "Unorder Packets(c2s)" + }, + { + "name": "common_s2c_tcp_unorder_num", + "type": "long", + "label": "Unorder Packets(s2c)" + }, + { + "name": "common_c2s_pkt_retrans", + "type": "long", + "label": "Packet Retransmission(c2s)" + }, + { + "name": "common_s2c_pkt_retrans", + "type": "long", + "label": "Packet Retransmission(s2c)" + }, + { + "name": "common_c2s_byte_retrans", + "type": "long", + "label": "Byte Retransmission(c2s)" + }, + { + "name": "common_s2c_byte_retrans", + "type": "long", + "label": "Byte Retransmission(s2c)" + }, + { + "name": "common_tcp_client_isn", + "type": "long", + "doc": { + "allow_query": "true" + }, + "label": "TCP Client ISN" + }, + { + "name": "common_tcp_server_isn", + "type": "long", + "doc": { + "allow_query": "true" + }, + "label": "TCP Server ISN" + }, + { + "name": "common_first_ttl", + "type": "int", + "doc": { + "visibility": "hidden" + }, + "label": "First TTL" + }, + { + "name": "common_processing_time", + "type": "long", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "label": "Processing Time" + }, + { + "name": "http_url", + "type": "string", + "label": "HTTP.URL" + }, + { + "name": "http_host", + "type": "string", + "doc": { + "format": { + "functions": "sub_domain", + "appendTo": "http_domain" + } + }, + "label": "HTTP.Host" + }, + { + "name": "http_domain", + "type": "string", + "doc": { + "allow_query": "true" + }, + "label": "HTTP.Domain" + }, + { + "name": "http_request_line", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "HTTP.Request Line" + }, + { + "name": "http_response_line", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "HTTP.Response Line" + }, + { + "name": "http_request_header", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Request Headers" + }, + { + "name": "http_response_header", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Response Headers" + }, + { + "name": "http_request_content", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Request Content" + }, + { + "name": "http_response_content", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Response Content" + }, + { + "name": "http_request_body", + "type": "string", + "doc": { + "constraints": { + "type": "file" + } + }, + "label": "HTTP.Request Body" + }, + { + "name": "http_response_body", + "type": "string", + "doc": { + "constraints": { + "type": "file" + } + }, + "label": "HTTP.Response Body" + }, + { + "name": "http_request_body_key", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "HTTP.Request Body Key" + }, + { + "name": "http_response_body_key", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "HTTP.Response Body Key" + }, + { + "name": "http_proxy_flag", + "type": "int", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Proxy Flag" + }, + { + "name": "http_sequence", + "type": "int", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Sequence" + }, + { + "name": "http_snapshot", + "type": "string", + "doc": { + "visibility": "hidden" + }, + "label": "HTTP.Snapshot" + }, + { + "name": "http_cookie", + "type": "string", + "label": "HTTP.Cookie" + }, + { + "name": "http_referer", + "type": "string", + "label": "HTTP.Referer" + }, + { + "name": "http_user_agent", + "type": "string", + "label": "HTTP.User Agent" + }, + { + "name": "http_content_length", + "type": "string", + "label": "HTTP.Content Length" + }, + { + "name": "http_content_type", + "type": "string", + "label": "HTTP.Content Type" + }, + { + "name": "http_set_cookie", + "type": "string", + "label": "HTTP.Set Cookie" + }, + { + "name": "http_version", + "type": "string", + "label": "HTTP.Version" + }, + { + "name": "http_response_latency_ms", + "type": "long", + "label": "HTTP.Response Latency(ms)" + }, + { + "name": "http_session_duration_ms", + "type": "long", + "label": "HTTP.Session Duration(ms)" + }, + { + "name": "http_action_file_size", + "type": "int", + "label": "HTTP.Action File Size" + }, + { + "name": "dns_message_id", + "type": "int", + "label": "DNS.Message ID" + }, + { + "name": "dns_qr", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "RESPONSE" + } + ] + }, + "label": "DNS.QR" + }, + { + "name": "dns_opcode", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "IQUERY" + }, + { + "code": "2", + "value": "STATUS" + }, + { + "code": "5", + "value": "UPDATE" + } + ] + }, + "label": "DNS.OPCODE" + }, + { + "name": "dns_aa", + "type": "int", + "label": "DNS.AA" + }, + { + "name": "dns_tc", + "type": "int", + "label": "DNS.TC" + }, + { + "name": "dns_rd", + "type": "int", + "label": "DNS.RD" + }, + { + "name": "dns_ra", + "type": "int", + "label": "DNS.RA" + }, + { + "name": "dns_rcode", + "type": "int", + "label": "DNS.RCODE" + }, + { + "name": "dns_qdcount", + "type": "int", + "label": "DNS.QDCOUNT" + }, + { + "name": "dns_ancount", + "type": "int", + "label": "DNS.ANCOUNT" + }, + { + "name": "dns_nscount", + "type": "int", + "label": "DNS.NSCOUNT" + }, + { + "name": "dns_arcount", + "type": "int", + "label": "DNS.ARCOUNT" + }, + { + "name": "dns_qname", + "type": "string", + "label": "DNS.QNAME" + }, + { + "name": "dns_qtype", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "A" + }, + { + "code": "2", + "value": "NS" + }, + { + "code": "5", + "value": "CNAME" + }, + { + "code": "6", + "value": "SOA" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "12", + "value": "PTR" + }, + { + "code": "13", + "value": "HINFO" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "15", + "value": "MX" + }, + { + "code": "28", + "value": "AAAA" + } + ] + }, + "label": "DNS.QTYPE" + }, + { + "name": "dns_qclass", + "type": "int", + "label": "DNS.QCLASS" + }, + { + "name": "dns_cname", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "DNS.CNAME" + }, + { + "name": "dns_sub", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "DNS" + }, + { + "code": "2", + "value": "DNSSEC" + } + ] + }, + "label": "DNS.SUB" + }, + { + "name": "dns_rr", + "type": "string", + "doc": { + "visibility": "disabled" + }, + "label": "DNS.RR" + }, + { + "name": "sip_call_id", + "type": "string", + "label": "SIP.Call-ID" + }, + { + "name": "sip_originator_description", + "type": "string", + "label": "SIP.Originator" + }, + { + "name": "sip_responder_description", + "type": "string", + "label": "SIP.Responder" + }, + { + "name": "sip_user_agent", + "type": "string", + "label": "SIP.User-Agent" + }, + { + "name": "sip_server", + "type": "string", + "label": "SIP.Server" + }, + { + "name": "sip_originator_sdp_connect_ip", + "type": "string", + "label": "SIP.Originator IP" + }, + { + "name": "sip_originator_sdp_media_port", + "type": "int", + "label": "SIP.Originator Port" + }, + { + "name": "sip_originator_sdp_media_type", + "type": "string", + "label": "SIP.Originator Media Type" + }, + { + "name": "sip_originator_sdp_content", + "type": "string", + "label": "SIP.Originator Content" + }, + { + "name": "sip_responder_sdp_connect_ip", + "type": "string", + "label": "SIP.Responder IP" + }, + { + "name": "sip_responder_sdp_media_port", + "type": "int", + "label": "SIP.Responder Port" + }, + { + "name": "sip_responder_sdp_media_type", + "type": "string", + "label": "SIP.Responder Media Type" + }, + { + "name": "sip_responder_sdp_content", + "type": "string", + "label": "SIP.Responder Content" + }, + { + "name": "sip_duration", + "type": "int", + "label": "SIP.Duration" + }, + { + "name": "sip_bye", + "type": "string", + "label": "SIP.Bye" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/version.json b/parcels/roles/services/galaxy-qgw-service/files/config/version.json new file mode 100644 index 0000000..4dd1f6a --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/version.json @@ -0,0 +1,95 @@ +{ + "product": "185 Environment", + "version": "21.07", + "registered": "Geedge", + "updated": "2021-08-01 00:00:00", + "components": { + "oss": [ + { + "name": "zookeeper", + "version": "3.4.10", + "licenseType": "Apache License 2.0", + "description": "分布式应用程序协调服务" + }, + { + "name": "kafka", + "version": "2.11_1.0.0", + "licenseType": "Apache License 2.0", + "description": "消息队列" + }, + { + "name": "habse", + "version": "2.2.3", + "licenseType": "Apache License 2.0", + "description": "用于文件系统和存储Radius数据" + }, + { + "name": "flume", + "version": "1.9.0", + "licenseType": "Apache License 2.0", + "description": "日志补全传输" + }, + { + "name": "clickhouse", + "version": "20.3.12.112-cluster", + "licenseType": "Apache License 2.0", + "description": "原始日志数据库" + }, + { + "name": "druid", + "version": "0.18.1", + "licenseType": "Apache License 2.0", + "description": "分析实时数据并提供低延迟查询的OLAP应用程序" + }, + { + "name": "gohangout", + "version": "1.15.2.20210408", + "description": "动态获取原始日志表schema入库程序" + } + ], + "apps": [ + { + "name": "galaxy-qgw-service", + "version": "345", + "description": "数据平台对外统一查询网关" + }, + { + "name": "galaxy-report-service", + "version": "21.04.07", + "description": "自定义报表查询服务" + }, + { + "name": "galaxy-hos-service", + "version": "21.07.01", + "description": "对象存储服务" + }, + { + "name": "xxl-job-admin", + "version": "v1.3.20210408", + "description": "分布式任务调度平台" + }, + { + "name": "xxl-job", + "version": "v1.3.210413-rc1", + "description": "分布式任务调度平台-执行器" + } + ], + "tasks": [ + { + "name": "flume", + "version": "flume-config-20.08", + "description": "原始日志补全、subscriber更新、Radius上下线功能" + }, + { + "name": "druid", + "version": "druid-config-20.08", + "description": "所有分析日志任务" + }, + { + "name": "gohangout", + "version": "gohangout-config-20.08", + "description": "原始日志入库、上下线日志入库" + } + ] + } +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/config/voip_record_log.json b/parcels/roles/services/galaxy-qgw-service/files/config/voip_record_log.json new file mode 100644 index 0000000..3691e86 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/config/voip_record_log.json @@ -0,0 +1,1992 @@ +{ + "type": "record", + "name": "voip_record_log", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "common_log_id", + "partition_key": "common_recv_time", + "functions": { + "aggregation": [ + { + "name": "COUNT", + "label": "COUNT", + "function": "count(expr)" + }, + { + "name": "COUNT_DISTINCT", + "label": "COUNT_DISTINCT", + "function": "count(distinct expr)" + }, + { + "name": "AVG", + "label": "AVG", + "function": "avg(expr)" + }, + { + "name": "SUM", + "label": "SUM", + "function": "sum(expr)" + }, + { + "name": "MAX", + "label": "MAX", + "function": "max(expr)" + }, + { + "name": "MIN", + "label": "MIN", + "function": "min(expr)" + } + ], + "operator": [ + { + "name": "=", + "label": "=", + "function": "expr = value" + }, + { + "name": "!=", + "label": "!=", + "function": "expr != value" + }, + { + "name": ">", + "label": ">", + "function": "expr > value" + }, + { + "name": "<", + "label": "<", + "function": "expr < value" + }, + { + "name": ">=", + "label": ">=", + "function": "expr >= value" + }, + { + "name": "<=", + "label": "<=", + "function": "expr <= value" + }, + { + "name": "has", + "label": "HAS", + "function": "has(expr, value)" + }, + { + "name": "in", + "label": "IN", + "function": "expr in (values)" + }, + { + "name": "not in", + "label": "NOT IN", + "function": "expr not in (values)" + }, + { + "name": "like", + "label": "LIKE", + "function": "expr like value" + }, + { + "name": "not like", + "label": "NOT LIKE", + "function": "expr not like value" + }, + { + "name": "notEmpty", + "label": "NOT EMPTY", + "function": "notEmpty(expr)" + }, + { + "name": "empty", + "label": "EMPTY", + "function": "empty(expr)" + } + ] + }, + "schema_query": { + "dimensions": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_sled_ip", + "common_device_id", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_client_port", + "common_server_port", + "common_schema_type", + "common_l4_protocol", + "common_l7_protocol", + "common_data_center", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_duration", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_originator_dir" + ], + "metrics": [ + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_subscriber_id", + "common_sled_ip", + "common_device_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_sessions", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_duration" + ], + "filters": [ + "common_address_type", + "common_server_ip", + "common_client_ip", + "common_internal_ip", + "common_external_ip", + "common_client_port", + "common_server_port", + "common_client_location", + "common_server_location", + "common_subscriber_id", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_l4_protocol", + "common_l7_protocol", + "common_stream_dir", + "common_direction", + "common_data_center", + "common_sled_ip", + "common_device_id", + "common_schema_type", + "common_client_asn", + "common_server_asn", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_duration", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_originator_dir" + ], + "references": { + "aggregation": [ + { + "type": "int", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "long", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "float", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "double", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN" + }, + { + "type": "string", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "date", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + } + ], + "operator": [ + { + "type": "int", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "long", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "float", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "double", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "string", + "functions": "=,!=,in,not in,like,not like,notEmpty,empty" + }, + { + "type": "date", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "timestamp", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "array", + "functions": "has" + } + ] + }, + "details": { + "general": [ + "common_recv_time", + "common_log_id", + "common_stream_trace_id", + "common_direction", + "common_stream_dir", + "common_start_time", + "common_end_time", + "common_con_duration_ms", + "common_establish_latency_ms", + "common_processing_time", + "common_entrance_id", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_data_center", + "common_sled_ip" + ], + "action": [ + "common_action", + "common_sub_action", + "common_policy_id", + "common_user_tags", + "common_user_region" + ], + "source": [ + "common_client_ip", + "common_internal_ip", + "common_client_port", + "common_client_location", + "common_client_asn", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number" + ], + "destination": [ + "common_server_ip", + "common_external_ip", + "common_server_port", + "common_server_location", + "common_server_asn" + ], + "application": [ + "common_app_id", + "common_userdefine_app_name", + "common_app_label", + "common_app_surrogate_id", + "common_l7_protocol", + "common_protocol_label", + "common_service_category", + "common_service", + "common_l4_protocol" + ], + "transmission": [ + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_first_ttl", + "common_tcp_client_isn", + "common_tcp_server_isn" + ], + "other": [ + "common_address_type", + "common_schema_type", + "common_device_tag", + "common_encapsulation", + "common_tunnels", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_link_info_c2s", + "common_link_info_s2c" + ] + } + }, + "schema_type": { + "SIP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration", + "sip_bye" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "common_server_ip", + "common_server_port" + ] + }, + "RTP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "common_server_ip", + "common_server_port", + "rtp_pcap_path", + "rtp_originator_dir" + ] + }, + "VoIP": { + "columns": [ + "common_recv_time", + "common_log_id", + "common_policy_id", + "common_subscriber_id", + "common_imei", + "common_imsi", + "common_phone_number", + "common_client_ip", + "common_client_port", + "common_internal_ip", + "common_l4_protocol", + "common_address_type", + "common_server_ip", + "common_server_port", + "common_external_ip", + "common_action", + "common_direction", + "common_entrance_id", + "common_sled_ip", + "common_client_location", + "common_client_asn", + "common_server_location", + "common_server_asn", + "common_sessions", + "common_c2s_pkt_num", + "common_s2c_pkt_num", + "common_c2s_byte_num", + "common_s2c_byte_num", + "common_c2s_pkt_diff", + "common_s2c_pkt_diff", + "common_c2s_byte_diff", + "common_s2c_byte_diff", + "common_service", + "common_schema_type", + "common_user_tags", + "common_sub_action", + "common_user_region", + "common_device_id", + "common_egress_link_id", + "common_ingress_link_id", + "common_isp", + "common_device_tag", + "common_data_center", + "common_encapsulation", + "common_app_label", + "common_tunnels", + "common_protocol_label", + "common_app_id", + "common_userdefine_app_name", + "common_app_surrogate_id", + "common_l7_protocol", + "common_service_category", + "common_start_time", + "common_end_time", + "common_establish_latency_ms", + "common_con_duration_ms", + "common_stream_dir", + "common_address_list", + "common_has_dup_traffic", + "common_stream_error", + "common_stream_trace_id", + "common_link_info_c2s", + "common_link_info_s2c", + "common_c2s_ipfrag_num", + "common_s2c_ipfrag_num", + "common_c2s_tcp_lostlen", + "common_s2c_tcp_lostlen", + "common_c2s_tcp_unorder_num", + "common_s2c_tcp_unorder_num", + "common_c2s_pkt_retrans", + "common_s2c_pkt_retrans", + "common_c2s_byte_retrans", + "common_s2c_byte_retrans", + "common_tcp_client_isn", + "common_tcp_server_isn", + "common_first_ttl", + "common_processing_time", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir" + ], + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "common_server_ip", + "common_server_port", + "rtp_pcap_path", + "rtp_originator_dir" + ] + } + }, + "default_columns": [ + "common_recv_time", + "common_log_id", + "common_subscriber_id", + "common_client_ip", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "common_server_ip", + "common_server_port", + "rtp_pcap_path", + "rtp_originator_dir" + ], + "tunnel_type": { + "GTP": [ + { + "name": "gtp_sgw_ip", + "label": "S-GW IP", + "type": "string" + }, + { + "name": "gtp_pgw_ip", + "label": "P-GW IP", + "type": "string" + }, + { + "name": "gtp_sgw_port", + "label": "S-GW Port", + "type": "int" + }, + { + "name": "gtp_pgw_port", + "label": "P-GW Port", + "type": "int" + }, + { + "name": "gtp_uplink_teid", + "label": "Uplink TEID", + "type": "long" + }, + { + "name": "gtp_downlink_teid", + "label": "Downlink TEID", + "type": "long" + } + ], + "MPLS": [ + { + "name": "mpls_c2s_direction_label", + "label": "Multiprotocol Label(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "mpls_s2c_direction_label", + "label": "Multiprotocol Label(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "VLAN": [ + { + "name": "vlan_c2s_direction_id", + "label": "VLAN Direction(c2s)", + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "vlan_s2c_direction_id", + "label": "VLAN Direction(s2c)", + "type": { + "type": "array", + "items": "int" + } + } + ], + "ETHERNET": [ + { + "name": "source_mac", + "label": "Source MAC", + "type": "string" + }, + { + "name": "destination_mac", + "label": "Destination MAC", + "type": "string" + } + ], + "MULTIPATH_ETHERNET": [ + { + "name": "c2s_source_mac", + "label": "Source MAC(c2s)", + "type": "string" + }, + { + "name": "c2s_destination_mac", + "label": "Destination MAC(c2s)", + "type": "string" + }, + { + "name": "s2c_source_mac", + "label": "Source MAC(s2c)", + "type": "string" + }, + { + "name": "s2c_destination_mac", + "label": "Destination MAC(s2c)", + "type": "string" + } + ] + } + }, + "fields": [ + { + "name": "common_recv_time", + "label": "Receive Time", + "doc": { + "allow_query": "true", + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_log_id", + "label": "Log ID", + "doc": { + "allow_query": "true", + "format": { + "functions": "snowflake_id" + } + }, + "type": "long" + }, + { + "name": "common_policy_id", + "label": "Policy ID", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_subscriber_id", + "label": "Subscriber ID", + "doc": { + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_imei", + "label": "IMEI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_imsi", + "label": "IMSI", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_phone_number", + "label": "Phone Number", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_client_ip", + "label": "Client IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn,radius_match", + "appendTo": "common_client_asn,common_subscriber_id" + } + }, + "type": "string" + }, + { + "name": "common_internal_ip", + "label": "Internal IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=69,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_client_port", + "label": "Client Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_l4_protocol", + "label": "L4 Protocol", + "type": "string" + }, + { + "name": "common_address_type", + "label": "Address Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ] + }, + "type": "int" + }, + { + "name": "common_server_ip", + "label": "Server IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + }, + "format": { + "functions": "geo_asn", + "appendTo": "common_server_asn" + } + }, + "type": "string" + }, + { + "name": "common_server_port", + "label": "Server Port", + "doc": { + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_external_ip", + "label": "External IP", + "doc": { + "constraints": { + "type": "ip" + }, + "format": { + "functions": "if", + "param": "$.common_direction=73,$.common_client_ip,$.common_server_ip" + }, + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_action", + "label": "Action", + "doc": { + "visibility": "hidden", + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "None" + }, + { + "code": "1", + "value": "Monitor" + }, + { + "code": "2", + "value": "Intercept" + }, + { + "code": "16", + "value": "Deny" + }, + { + "code": "128", + "value": "Allow" + } + ] + }, + "type": "int" + }, + { + "name": "common_direction", + "label": "Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "69", + "value": "outbound" + }, + { + "code": "73", + "value": "inbound" + } + ] + }, + "type": "int" + }, + { + "name": "common_entrance_id", + "label": "Entrance ID", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_sled_ip", + "label": "Sled IP", + "doc": { + "allow_query": "true", + "constraints": { + "type": "ip" + } + }, + "type": "string" + }, + { + "name": "common_client_location", + "label": "Client Location", + "type": "string" + }, + { + "name": "common_client_asn", + "label": "Client ASN", + "type": "string" + }, + { + "name": "common_server_location", + "label": "Server Location", + "type": "string" + }, + { + "name": "common_server_asn", + "label": "Server ASN", + "type": "string" + }, + { + "name": "common_sessions", + "label": "Sessions", + "type": "long" + }, + { + "name": "common_c2s_pkt_num", + "label": "Packets Sent", + "type": "long" + }, + { + "name": "common_s2c_pkt_num", + "label": "Packets Received", + "type": "long" + }, + { + "name": "common_c2s_byte_num", + "label": "Bytes Sent", + "type": "long" + }, + { + "name": "common_s2c_byte_num", + "label": "Bytes Received", + "type": "long" + }, + { + "name": "common_c2s_pkt_diff", + "label": "Packets Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_diff", + "label": "Packets Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_diff", + "label": "Bytes Sent(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_diff", + "label": "Bytes Received(Diff)", + "doc": { + "visibility": "disabled" + }, + "type": "long" + }, + { + "name": "common_service", + "label": "Service", + "doc": { + "visibility": "disabled" + }, + "type": "int" + }, + { + "name": "common_schema_type", + "label": "Schema Type", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "VoIP", + "value": "VoIP" + }, + { + "code": "SIP", + "value": "SIP" + }, + { + "code": "RTP", + "value": "RTP" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_user_tags", + "label": "User Tags", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_sub_action", + "label": "Sub Action", + "doc": { + "data": [ + { + "code": "allow", + "value": "Allow" + }, + { + "code": "deny", + "value": "Deny" + }, + { + "code": "monitor", + "value": "Monitor" + }, + { + "code": "replace", + "value": "Replace" + }, + { + "code": "redirect", + "value": "Redirect" + }, + { + "code": "insert", + "value": "Insert" + }, + { + "code": "hijack", + "value": "Hijack" + } + ], + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_user_region", + "label": "User Region", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_device_id", + "label": "Device ID", + "type": "string" + }, + { + "name": "common_egress_link_id", + "label": "Egress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_ingress_link_id", + "label": "Ingress Link ID", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_isp", + "label": "ISP", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_device_tag", + "label": "Device Tag", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_data_center", + "label": "Data Center", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "City A", + "value": "City A" + }, + { + "code": "City B", + "value": "City B" + }, + { + "code": "City C", + "value": "City C" + }, + { + "code": "City D", + "value": "City D" + }, + { + "code": "City E", + "value": "City E" + }, + { + "code": "City F", + "value": "City F" + }, + { + "code": "City G", + "value": "City G" + }, + { + "code": "City H", + "value": "City H" + }, + { + "code": "City I", + "value": "City I" + }, + { + "code": "City J", + "value": "City J" + }, + { + "code": "City K", + "value": "City K" + }, + { + "code": "City L", + "value": "City L" + }, + { + "code": "City M", + "value": "City M" + }, + { + "code": "City N", + "value": "City N" + }, + { + "code": "City O", + "value": "City O" + }, + { + "code": "City P", + "value": "City P" + }, + { + "code": "City Q", + "value": "City Q" + }, + { + "code": "City R", + "value": "City R" + } + ], + "allow_query": "true" + }, + "type": "string" + }, + { + "name": "common_encapsulation", + "label": "Encapsulation", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "Ethernet" + }, + { + "code": "8", + "value": "PPP" + }, + { + "code": "12", + "value": "CiscoHDLC" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_app_label", + "label": "Application Label", + "type": "string" + }, + { + "name": "common_tunnels", + "label": "Tunnels", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_protocol_label", + "label": "Protocol Label", + "type": "string" + }, + { + "name": "common_app_id", + "label": "Application ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_userdefine_app_name", + "label": "User Define APP Name", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_app_surrogate_id", + "label": "Surrogate ID", + "type": "string", + "doc": { + "visibility": "hidden" + } + }, + { + "name": "common_l7_protocol", + "label": "L7 Protocol", + "type": "string" + }, + { + "name": "common_service_category", + "label": "FQDN Category", + "doc": { + "constraints": { + "operator_functions": "has" + }, + "visibility": "disabled", + "dict_location": { + "path": "/v1/category/dict", + "key": "categoryId", + "value": "categoryName" + } + }, + "type": { + "type": "array", + "items": "int" + } + }, + { + "name": "common_start_time", + "label": "Start Time", + "doc": { + "constraints": { + "type": "timestamp" + } + }, + "type": "long" + }, + { + "name": "common_end_time", + "label": "End Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "get_value", + "appendTo": "common_recv_time" + } + }, + "type": "long" + }, + { + "name": "common_establish_latency_ms", + "label": "Establish Latency(ms)", + "type": "long" + }, + { + "name": "common_con_duration_ms", + "label": "Duration(ms)", + "type": "long" + }, + { + "name": "common_stream_dir", + "label": "Stream Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + }, + { + "code": "3", + "value": "double" + } + ], + "allow_query": "true" + }, + "type": "int" + }, + { + "name": "common_address_list", + "label": "Address List", + "doc": { + "visibility": "disabled" + }, + "type": "string" + }, + { + "name": "common_has_dup_traffic", + "label": "Duplication Traffic", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ], + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_stream_error", + "label": "Stream Error", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_stream_trace_id", + "label": "Session ID", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_link_info_c2s", + "label": "Link Info(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_link_info_s2c", + "label": "Link Info(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "string" + }, + { + "name": "common_c2s_ipfrag_num", + "label": "Fragmentation Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_ipfrag_num", + "label": "Fragmentation Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_lostlen", + "label": "Sequence Gap Loss(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_lostlen", + "label": "Sequence Gap Loss(s2c)", + "type": "long" + }, + { + "name": "common_c2s_tcp_unorder_num", + "label": "Unorder Packets(c2s)", + "type": "long" + }, + { + "name": "common_s2c_tcp_unorder_num", + "label": "Unorder Packets(s2c)", + "type": "long" + }, + { + "name": "common_c2s_pkt_retrans", + "label": "Packet Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_pkt_retrans", + "label": "Packet Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_c2s_byte_retrans", + "label": "Byte Retransmission(c2s)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_s2c_byte_retrans", + "label": "Byte Retransmission(s2c)", + "doc": { + "visibility": "hidden" + }, + "type": "long" + }, + { + "name": "common_tcp_client_isn", + "label": "TCP Client ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_tcp_server_isn", + "label": "TCP Server ISN", + "doc": { + "allow_query": "true" + }, + "type": "long" + }, + { + "name": "common_first_ttl", + "label": "First TTL", + "doc": { + "visibility": "hidden" + }, + "type": "int" + }, + { + "name": "common_processing_time", + "label": "Processing Time", + "doc": { + "constraints": { + "type": "timestamp" + }, + "format": { + "functions": "current_timestamp" + } + }, + "type": "long" + }, + { + "name": "sip_call_id", + "label": "SIP.Call-ID", + "type": "string" + }, + { + "name": "sip_originator_description", + "label": "SIP.Originator", + "type": "string" + }, + { + "name": "sip_responder_description", + "label": "SIP.Responder", + "type": "string" + }, + { + "name": "sip_user_agent", + "label": "SIP.User-Agent", + "type": "string" + }, + { + "name": "sip_server", + "label": "SIP.Server", + "type": "string" + }, + { + "name": "sip_originator_sdp_connect_ip", + "label": "SIP.Originator IP", + "type": "string" + }, + { + "name": "sip_originator_sdp_media_port", + "label": "SIP.Originator Port", + "type": "int" + }, + { + "name": "sip_originator_sdp_media_type", + "label": "SIP.Originator Media Type", + "type": "string" + }, + { + "name": "sip_originator_sdp_content", + "label": "SIP.Originator Content", + "type": "string" + }, + { + "name": "sip_responder_sdp_connect_ip", + "label": "SIP.Responder IP", + "type": "string" + }, + { + "name": "sip_responder_sdp_media_port", + "label": "SIP.Responder Port", + "type": "int" + }, + { + "name": "sip_responder_sdp_media_type", + "label": "SIP.Responder Media Type", + "type": "string" + }, + { + "name": "sip_responder_sdp_content", + "label": "SIP.Responder Content", + "type": "string" + }, + { + "name": "sip_duration", + "label": "SIP.Duration", + "type": "int" + }, + { + "name": "sip_bye", + "label": "SIP.Bye", + "type": "string" + }, + { + "name": "rtp_payload_type_c2s", + "label": "RTP.Payload Type(c2s)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_payload_type_s2c", + "label": "RTP.Payload Type(s2c)", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ] + }, + "type": "int" + }, + { + "name": "rtp_pcap_path", + "label": "RTP.PCAP", + "doc": { + "constraints": { + "type": "files" + } + }, + "type": "string" + }, + { + "name": "rtp_originator_dir", + "label": "RTP.Direction", + "doc": { + "constraints": { + "operator_functions": "=,!=" + }, + "data": [ + { + "code": "0", + "value": "unknown" + }, + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + } + ] + }, + "type": "int" + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/files/dat/asn_v4.mmdb b/parcels/roles/services/galaxy-qgw-service/files/dat/asn_v4.mmdb Binary files differnew file mode 100644 index 0000000..68d267c --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/dat/asn_v4.mmdb diff --git a/parcels/roles/services/galaxy-qgw-service/files/dat/asn_v6.mmdb b/parcels/roles/services/galaxy-qgw-service/files/dat/asn_v6.mmdb Binary files differnew file mode 100644 index 0000000..ad5f669 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/dat/asn_v6.mmdb diff --git a/parcels/roles/services/galaxy-qgw-service/files/dat/ip_private_v4.mmdb b/parcels/roles/services/galaxy-qgw-service/files/dat/ip_private_v4.mmdb Binary files differnew file mode 100644 index 0000000..445f2aa --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/dat/ip_private_v4.mmdb diff --git a/parcels/roles/services/galaxy-qgw-service/files/dat/ip_private_v6.mmdb b/parcels/roles/services/galaxy-qgw-service/files/dat/ip_private_v6.mmdb Binary files differnew file mode 100644 index 0000000..cf82ebe --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/dat/ip_private_v6.mmdb diff --git a/parcels/roles/services/galaxy-qgw-service/files/dat/ip_v4.mmdb b/parcels/roles/services/galaxy-qgw-service/files/dat/ip_v4.mmdb Binary files differnew file mode 100644 index 0000000..842cce9 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/dat/ip_v4.mmdb diff --git a/parcels/roles/services/galaxy-qgw-service/files/dat/ip_v6.mmdb b/parcels/roles/services/galaxy-qgw-service/files/dat/ip_v6.mmdb Binary files differnew file mode 100644 index 0000000..cf82ebe --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/files/dat/ip_v6.mmdb diff --git a/parcels/roles/services/galaxy-qgw-service/tasks/load_qgw.yml b/parcels/roles/services/galaxy-qgw-service/tasks/load_qgw.yml new file mode 100644 index 0000000..3a1c44d --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/tasks/load_qgw.yml @@ -0,0 +1,114 @@ +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181" + when: '(groups.zookeeper|length) == 3' + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181,{{groups.zookeeper[3]}}:2181,{{groups.zookeeper[4]}}:2181" + when: '(groups.zookeeper|length) >= 5' + +- name: stop and remove {{ qgw_image_container_name }} container + docker_container: + name: '{{ qgw_image_container_name }}' + state: absent + +- name: remove old {{ qgw_image_name }} image + docker_image: + name: '{{ qgw_image_name }}' + tag: '{{ qgw_image_tag_name }}' + state: absent + +- name: create {{ qgw_volume_path }} + file: + state: directory + path: '{{ install_path }}/{{ qgw_volume_path }}' + +- name: create {{ qgw_volume_path }}/config + file: + state: directory + path: '{{ install_path }}/{{ qgw_volume_path }}/config' + +- name: create {{ install_path }}/{{ qgw_soft_home_path }} + file: + state: directory + path: '{{ install_path }}/{{ qgw_soft_home_path }}' + +- name: copy {{ qgw_image_tar_name }} + copy: + src: '{{ package_path }}/{{ qgw_image_tar_name }}' + dest: '{{ install_path }}/{{ qgw_soft_home_path }}/{{ qgw_image_tar_name }}' + force: true + backup: yes + +- name: load new {{ qgw_image_name }} image from {{ qgw_image_tar_name }} + docker_image: + name: '{{ qgw_image_name }}' + tag: '{{ qgw_image_tag_name }}' + load_path: '{{ install_path }}/{{ qgw_soft_home_path }}/{{ qgw_image_tar_name }}' + source: load + force_tag: yes + force_source: yes + +- name: change the image tag + shell: docker tag galaxy-qgw-service:{{ qgw_image_tag_name }} {{ qgw_image_name }}:{{ qgw_image_tag_name }} + +- name: copy {{ qgw_image_name }} docker-compose.yml + template: + src: docker-compose.yml.j2 + dest: '{{ install_path }}/{{ qgw_soft_home_path }}/docker-compose.yml' + backup: yes + +- name: copy {{ qgw_image_name }} docker-compose.yml + template: + src: application.yml.j2 + dest: '{{ install_path }}/{{ qgw_volume_path }}/config/application.yml' + backup: yes + +- name: copy {{ qgw_image_name }} docker-compose.yml + template: + src: log4j2.xml.j2 + dest: '{{ install_path }}/{{ qgw_volume_path }}/config/log4j2.xml' + backup: yes + +- name: copy dat to {{ install_path }}/{{ qgw_volume_path }}/ + copy: src=files/dat dest={{ install_path }}/{{ qgw_volume_path }}/ force=true + +- name: copy config to {{ install_path }}/{{ qgw_soft_home_path }}/ + copy: src=files/config dest={{ install_path }}/{{ qgw_soft_home_path }}/ force=true + +- name: copy {{ qgw_image_name }} docker-compose.yml + template: + src: memorySchema.json.j2 + dest: '{{ install_path }}/{{ qgw_volume_path }}/config/memorySchema.json' + backup: yes + +- name: copy galaxy-qgw-service.j2 to {{ install_path }}/{{ qgw_soft_home_path }}/ + template: + src: galaxy-qgw-service.yml.j2 + dest: '{{ install_path }}/{{ qgw_soft_home_path }}/config/galaxy-qgw-service.yml' + backup: yes + +#- name: push config +# shell: 'curl --data-urlencode content="`cat {{ install_path }}/{{ qgw_soft_home_path }}/galaxy-qgw-service`" --request POST "http://{{ gateway_keepalive_host }}:8848/nacos/v1/cs/configs?username=nacos&password={{ nacos_pin }}&tenant={{ services_config_namespace }}&group=Galaxy&dataId=galaxy-qgw-service.yml&appName=galaxy-qgw-service&type=yaml"' +# register: change_out + +#- debug: +# msg: '{{ change_out.stdout }}' + +#- fail: +# msg: "galaxy-qgw-admin 配置提交失败" +# when: change_out.stdout != 'true' + +- name: copy push_config.sh.j2 to {{ install_path }}/{{ qgw_soft_home_path }}/ + template: + src: push_config.sh.j2 + dest: '{{ install_path }}/{{ qgw_soft_home_path }}/push_config.sh' + mode: 0755 + force: yes + +- name: push config + shell: 'cd {{ install_path }}/{{ qgw_soft_home_path }} && chmod +x push_config.sh && sh push_config.sh' + +- name: start {{ qgw_image_container_name }} container + docker_compose: + project_src: '{{ install_path }}/{{ qgw_soft_home_path }}' + diff --git a/parcels/roles/services/galaxy-qgw-service/tasks/main.yml b/parcels/roles/services/galaxy-qgw-service/tasks/main.yml new file mode 100644 index 0000000..64ebe7c --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/tasks/main.yml @@ -0,0 +1,2 @@ +#- include: update_python3_and_docker_py.yml +- include: load_qgw.yml diff --git a/parcels/roles/services/galaxy-qgw-service/templates/application.yml.j2 b/parcels/roles/services/galaxy-qgw-service/templates/application.yml.j2 new file mode 100644 index 0000000..53544f1 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/templates/application.yml.j2 @@ -0,0 +1,20 @@ +nacos: + config: + type: yaml #配置集的配置格式 + server-addr: {{ gateway_keepalive_host }}:8848 #配置中心地址 + namespace: {{ services_config_namespace }} #命名空间 + data-id: galaxy-qgw-service.yml # 数据集ID + auto-refresh: true #开启自动刷新 + group: Galaxy #配置对应的分组 + username: nacos #Nacos认证用户 + password: {{ nacos_pin }} #Nacos认证密码 + bootstrap: + enable: true #开启配置预加载功能 + log: + enable: true #开启Nacos支持日志级别的加载时机 +spring: + profiles: + active: prod +logging: + config: ./config/log4j2.xml + diff --git a/parcels/roles/services/galaxy-qgw-service/templates/docker-compose.yml.j2 b/parcels/roles/services/galaxy-qgw-service/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..fa3d663 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/templates/docker-compose.yml.j2 @@ -0,0 +1,19 @@ +version: "3" +services: + #设置容器名称和docker-compose中的名称(文件夹名称_当前名称_数字),如果下面配置了container_name,以container_name中的值为准 + galaxy-qgw-service: + #依赖的镜像 + image: {{ qgw_image_name }}:{{ qgw_image_tag_name }} + environment: + JAVA_OPTS: "{{ qgw_java_opts }}" + container_name: {{ qgw_image_container_name }} + ports: + - "8183:8183" + volumes: + - "{{ install_path }}/{{ qgw_volume_path }}/logs:/home/tsg/galaxy/galaxy-qgw-service/logs" + - "{{ install_path }}/{{ qgw_volume_path }}/config:/home/tsg/galaxy/galaxy-qgw-service/config" + - "{{ install_path }}/{{ qgw_volume_path }}/dat:/home/tsg/galaxy/galaxy-qgw-service/dat" + - "/etc/localtime:/etc/localtime:ro" + - "/etc/timezone:/etc/timezone:ro" + restart: always + network_mode: "host" diff --git a/parcels/roles/services/galaxy-qgw-service/templates/galaxy-qgw-service.yml.j2 b/parcels/roles/services/galaxy-qgw-service/templates/galaxy-qgw-service.yml.j2 new file mode 100644 index 0000000..2a36583 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/templates/galaxy-qgw-service.yml.j2 @@ -0,0 +1,111 @@ +##############静态参数配置(修改后需要重启项目)############## +## 服务配置 +server: + port: 8183 +## 监控参数 +management: + metrics: + tags: + application: ${project.name} + endpoint: + health: + show-details: always + shutdown: + enabled: true + health: + redis: + enabled: false + db: + enabled: false + endpoints: + web: + exposure: + include: '*' + exclude: env,auditevents,beans,conditions,info + base-path: /monitor +#MySql configuration +spring: + datasource: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://{{ bifang_mariadb_host }}:3306/tsg-bifang?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=GMT + username: root + pin: {{ bifang_mariadb_pin }} +##############动态参数配置(修改后不需要重启项目)############## +## 项目参数 +project: + name: galaxy-qgw-service-nacos + description: 统一数据查询网关 + version: 1.1 + groupId: com.mesalab + artifactId: galaxy-qgw-service + basedir: + corePackage: com.mesalab.common + servicePackage: com.mesalab.qgw + author: + name: darnell + url: + email: +## ClickhHouse configuration +clickhouse: + url: http://{{ gateway_keepalive_host }}:8124 + dbname: tsg_galaxy_v3 + realTimeAccount: + username: tsg_query + pin: ceiec2018 + socketTimeOut: 60000 + longTermAccount: + username: tsg_report + pin: ceiec2019 + socketTimeOut: 21700000 +## Druid configuration +druid: + url: {{ gateway_keepalive_host }}:8089/druid/v2/sql + dbname: druid + socketTimeOut: 60000 +## Elasticsearch configuration +elasticsearch: + url: 127.0.0.1:9200/_sql + dbname: elasticsearch + socketTimeOut: 60000 +## Hbase configuration +hbase: + zookeeperQuorum: {{ zookeeper_servers }} + zookeeperZnodeParent: /hbase + dbname: tsg + tableName: report_result + columnFamily: response + columnName: result + clientIpcPoolSize: 60 + rpcTimeout: 60000 +## xxl-job-admin configuration +xxl-job-admin: + url: http://{{ gateway_keepalive_host }}:8181/xxl-job-admin + userName: admin + pin: galaxy_2019 +## Engine configuration +engine: + maxCacheNum: 1048576 + defaultResultNum: 100000 +## ArangoDB configuration +arango: + server: http://{{groups.arangodb[0]}}:8529 + database: tsg_galaxy_v3 + username: query + pin: ceiec2018 + jwturl: ${arango.server}/_db/${arango.database}/_open/auth + queryurl: ${arango.server}/_db/${arango.database}/_api/cursor + maxrows: 10000 + socketTimeOut: 300000 +## http pool config +http: + pool: + connect: + timeout: 30000 + max: + connection: 500 + per: + route: 200 + request: + timeout: 10000 + response: + timeout: 10000 diff --git a/parcels/roles/services/galaxy-qgw-service/templates/log4j2.xml.j2 b/parcels/roles/services/galaxy-qgw-service/templates/log4j2.xml.j2 new file mode 100644 index 0000000..5ec9494 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/templates/log4j2.xml.j2 @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="UTF-8"?> +<configuration> + + <!--日志打印相关参数配置--> + <Properties> + <!--每5M压缩日志文件--> + <property name="LOG_SIZE">5M</property> + <!--最多产生10个压缩文件--> + <property name="LOG_NUMS">10</property> + <!--日志打印等级--> + <property name="LOG_LEVEL">info</property> + <!--日志文件路径--> + <property name="LOG_PATH">logs</property> + <!--日志文件名称--> + <property name="LOG_FILE_NAME">galaxy-qgw-service</property> + <!--日志打印格式--> + <property name="LOG_PATTERN">[%d{yyyy-MM-dd HH:mm:ss}] [%p] [Thread:%t] %l %x - %m%n</property> + </Properties> + + <appenders> + <Console name="consoleSystemOutAppender" target="SYSTEM_OUT"> + <ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/> + <PatternLayout pattern="${LOG_PATTERN}"/> + </Console> + + <RollingFile name="rollingFileAllAppender" + fileName="${LOG_PATH}/${LOG_FILE_NAME}.log" + filePattern="${LOG_PATH}/history/$${date:yyyy-MM-dd}/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz"> + <PatternLayout pattern="${LOG_PATTERN}"/> + <Policies> + <SizeBasedTriggeringPolicy size="${LOG_SIZE}"/> + <TimeBasedTriggeringPolicy interval="1" modulate="true"/> + </Policies> + <Filters> + <ThresholdFilter level="all" onMatch="ACCEPT" onMismatch="DENY"/> + </Filters> + <DefaultRolloverStrategy max="${LOG_NUMS}"> + <Delete basePath="${LOG_PATH}/history" maxDepth="1"> + <IfFileName glob="*.log.gz"> + <IfLastModified age="90d"> + <IfAny> + <IfAccumulatedFileSize exceeds="200 GB" /> + </IfAny> + </IfLastModified> + </IfFileName> + </Delete> + </DefaultRolloverStrategy> + </RollingFile> + </appenders> + <loggers> + <root level="${LOG_LEVEL}"> + <appender-ref ref="consoleSystemOutAppender"/> + <appender-ref ref="rollingFileAllAppender"/> + </root> + </loggers> +</configuration>
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/templates/memorySchema.json.j2 b/parcels/roles/services/galaxy-qgw-service/templates/memorySchema.json.j2 new file mode 100644 index 0000000..a034ee1 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/templates/memorySchema.json.j2 @@ -0,0 +1,13 @@ +{ + "version": "1.0", + "defaultSchema": "memorySchema", + "schemas": [ + { + "type": "custom", + "name": "memorySchema", + "factory": "com.mesalab.calcite.InMemorySchemaFactory", + "operand": { + } + } + ] +}
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-qgw-service/templates/push_config.sh.j2 b/parcels/roles/services/galaxy-qgw-service/templates/push_config.sh.j2 new file mode 100755 index 0000000..c7c3643 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/templates/push_config.sh.j2 @@ -0,0 +1,49 @@ +#!/bin/bash + +config_path_root={{ install_path }}/{{ qgw_soft_home_path }}/config +tenant={{ services_config_namespace }} +nacos_push_url="http://{{ gateway_keepalive_host }}:8848/nacos/v1/cs/configs?username=nacos&password={{ nacos_pin }}&tenant=$tenant&group=Galaxy" +retry=10 + +#1.获取config 路径 +config_path_list=$(readlink -f $config_path_root) + +function pushConfig(){ +#2.app文件夹文件 +for config_path in $config_path_list +do + if [ -d $config_path ];then + appPath=${config_path%/*} + appName=${appPath##*/} + for file in `ls $config_path` + do + config_path_file=$config_path/$file + if [ -f $config_path_file ];then + data_id=${config_path_file##*/} + suffix=${config_path_file##*.} + if [ "$suffix" = "sql" ];then + suffix="text" + fi + if [ "$suffix" = "yml" ];then + suffix="yaml" + fi + + #3.提交config配置 + for (( i = 0; i < 10; i++ )); do + if [ "$suffix" = "yaml" ]||[ "$suffix" = "json" ]||[ "$suffix" = "text" ]||[ "$suffix" = "properties" ];then + result=$(curl -sw '%{http_code}' -o /dev/null --request POST "$nacos_push_url&dataId=$data_id&appName=$appName&type=$suffix" --data-urlencode content="`cat $config_path_file`") + echo "push config response code "$result + if [[ $result -eq '200' ]];then + i=10 + fi + fi + done + + fi + done + fi +done +} + +pushConfig + diff --git a/parcels/roles/services/galaxy-qgw-service/vars/main.yml b/parcels/roles/services/galaxy-qgw-service/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/services/galaxy-qgw-service/vars/main.yml diff --git a/parcels/roles/services/galaxy-report-service/files/config/log4j2-dev.xml b/parcels/roles/services/galaxy-report-service/files/config/log4j2-dev.xml new file mode 100644 index 0000000..94f8476 --- /dev/null +++ b/parcels/roles/services/galaxy-report-service/files/config/log4j2-dev.xml @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="UTF-8"?> +<configuration> + + <!--日志打印相关参数配置--> + <Properties> + <!--每5M压缩日志文件--> + <property name="LOG_SIZE">5M</property> + <!--最多产生10个压缩文件--> + <property name="LOG_NUMS">10</property> + <!--日志打印等级--> + <property name="LOG_LEVEL">info</property> + <!--日志文件路径--> + <property name="LOG_PATH">logs</property> + <!--日志文件名称--> + <property name="LOG_FILE_NAME">galaxy-report-service</property> + <!--日志打印格式--> + <property name="LOG_PATTERN">[%d{yyyy-MM-dd HH:mm:ss}] [%p] [Thread:%t] %l %x - %m%n</property> + </Properties> + + <appenders> + <Console name="consoleSystemOutAppender" target="SYSTEM_OUT"> + <ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/> + <PatternLayout pattern="${LOG_PATTERN}"/> + </Console> + + <RollingFile name="rollingFileAllAppender" + fileName="${LOG_PATH}/${LOG_FILE_NAME}.log" + filePattern="${LOG_PATH}/history/$${date:yyyy-MM-dd}/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz"> + <PatternLayout pattern="${LOG_PATTERN}"/> + <Policies> + <SizeBasedTriggeringPolicy size="${LOG_SIZE}"/> + <TimeBasedTriggeringPolicy interval="1" modulate="true"/> + </Policies> + <Filters> + <ThresholdFilter level="all" onMatch="ACCEPT" onMismatch="DENY"/> + </Filters> + <DefaultRolloverStrategy max="${LOG_NUMS}"> + <Delete basePath="${LOG_PATH}/history" maxDepth="1"> + <IfFileName glob="*.log.gz"> + <IfLastModified age="90d"> + <IfAny> + <IfAccumulatedFileSize exceeds="200 GB" /> + </IfAny> + </IfLastModified> + </IfFileName> + </Delete> + </DefaultRolloverStrategy> + </RollingFile> + </appenders> + <loggers> + <root level="${LOG_LEVEL}"> + <appender-ref ref="consoleSystemOutAppender"/> + <appender-ref ref="rollingFileAllAppender"/> + </root> + </loggers> +</configuration>
\ No newline at end of file diff --git a/parcels/roles/services/galaxy-report-service/tasks/load_report.yml b/parcels/roles/services/galaxy-report-service/tasks/load_report.yml new file mode 100644 index 0000000..b0a4e44 --- /dev/null +++ b/parcels/roles/services/galaxy-report-service/tasks/load_report.yml @@ -0,0 +1,75 @@ +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181" + when: '(groups.zookeeper|length) == 3' + +- name: get zookeeper_servers to ansible variable + set_fact: zookeeper_servers="{{groups.zookeeper[0]}}:2181,{{groups.zookeeper[1]}}:2181,{{groups.zookeeper[2]}}:2181,{{groups.zookeeper[3]}}:2181,{{groups.zookeeper[4]}}:2181" + when: '(groups.zookeeper|length) >= 5' + +- name: stop and remove {{ report_image_container_name }} container + docker_container: + name: '{{ report_image_container_name }}' + state: absent + +- name: remove old {{ report_image_name }} image + docker_image: + name: '{{ report_image_name }}' + tag: '{{ report_image_tag_name }}' + state: absent + +- name: create {{ install_path }}/{{ report_soft_home_path }} + file: + state: directory + path: '{{ install_path }}/{{ report_soft_home_path }}' + +- name: copy {{ report_image_tar_name }} + copy: + src: '{{ package_path }}/{{ report_image_tar_name }}' + dest: '{{ install_path }}/{{ report_soft_home_path }}/{{ report_image_tar_name }}' + force: true + backup: yes + +- name: copy config to {{ report_volume_path }} + copy: src=files/config dest={{ install_path }}/{{ report_volume_path }}/ force=true backup=yes + +- name: copy application.yml.j2 + template: + src: application.yml.j2 + dest: '{{ install_path }}/{{ report_volume_path }}/config/application.yml' + backup: yes + +- name: load new {{ report_image_name }} image from {{ report_image_tar_name }} + docker_image: + name: '{{ report_image_name }}' + tag: '{{ report_image_tag_name }}' + load_path: '{{ install_path }}/{{ report_soft_home_path }}/{{ report_image_tar_name }}' + source: load + force_tag: yes + force_source: yes + +- name: change the image tag + shell: docker tag galaxy-report-service:{{ report_image_tag_name }} {{ report_image_name }}:{{ report_image_tag_name }} + +- name: copy {{ report_image_name }} docker-compose.yml + template: + src: docker-compose.yml.j2 + dest: '{{ install_path }}/{{ report_soft_home_path }}/docker-compose.yml' + backup: yes + +- name: copy {{ report_image_name }} galaxy-report-service.j2 + template: + src: galaxy-report-service.j2 + dest: '{{ install_path }}/{{ report_soft_home_path }}/galaxy-report-service' + backup: yes + +- name: push config + shell: 'curl --data-urlencode content="`cat {{ install_path }}/{{ report_soft_home_path }}/galaxy-report-service`" --request POST "http://{{ gateway_keepalive_host }}:8848/nacos/v1/cs/configs?username=nacos&password={{ nacos_pin }}&tenant={{ services_config_namespace }}&group=Galaxy&dataId=galaxy-report-service.yml&appName=galaxy-report-service&type=yaml"' + register: change_out + +- fail: + msg: "galaxy-report-service 配置提交失败" + when: change_out.stdout != 'true' + +- name: start {{ report_image_container_name }} container + docker_compose: + project_src: '{{ install_path }}/{{ report_soft_home_path }}' diff --git a/parcels/roles/services/galaxy-report-service/tasks/main.yml b/parcels/roles/services/galaxy-report-service/tasks/main.yml new file mode 100644 index 0000000..dc3373a --- /dev/null +++ b/parcels/roles/services/galaxy-report-service/tasks/main.yml @@ -0,0 +1,2 @@ +#- include: update_python3_and_docker_py.yml +- include: load_report.yml diff --git a/parcels/roles/services/galaxy-report-service/templates/application.yml.j2 b/parcels/roles/services/galaxy-report-service/templates/application.yml.j2 new file mode 100644 index 0000000..baa9644 --- /dev/null +++ b/parcels/roles/services/galaxy-report-service/templates/application.yml.j2 @@ -0,0 +1,21 @@ +nacos: + config: + type: yaml #配置集的配置格式 + server-addr: {{ gateway_keepalive_host }}:8848 #配置中心地址 + namespace: {{ services_config_namespace }} #命名空间 + data-id: galaxy-report-service.yml # 数据集ID + auto-refresh: true #开启自动刷新 + group: Galaxy #配置对应的分组 + username: nacos #Nacos认证用户 + password: {{ nacos_pin }} #Nacos认证密码 + bootstrap: + enable: true #开启配置预加载功能 + log: + enable: true #开启Nacos支持日志级别的加载时机 +spring: + profiles: + active: dev +logging: + config: ./config/log4j2-dev.xml + + diff --git a/parcels/roles/services/galaxy-report-service/templates/docker-compose.yml.j2 b/parcels/roles/services/galaxy-report-service/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..022469c --- /dev/null +++ b/parcels/roles/services/galaxy-report-service/templates/docker-compose.yml.j2 @@ -0,0 +1,18 @@ +version: "3" +services: + #设置容器名称和docker-compose中的名称(文件夹名称_当前名称_数字),如果下面配置了container_name,以container_name中的值为准 + galaxy-report-service: + #依赖的镜像 + image: {{ report_image_name }}:{{ report_image_tag_name }} + container_name: {{ report_image_container_name }} + environment: + JAVA_OPTS: "{{ report_java_opts }}" + ports: + - "9093:9093" + volumes: + - "{{ install_path }}/{{ report_volume_path }}/config:/home/tsg/galaxy/galaxy-report-service/config" + - "{{ install_path }}/{{ report_volume_path }}/logs:/home/tsg/galaxy/galaxy-report-service/logs" + - "/etc/localtime:/etc/localtime:ro" + - "/etc/timezone:/etc/timezone:ro" + restart: always + network_mode: 'host' diff --git a/parcels/roles/services/galaxy-report-service/templates/galaxy-report-service.j2 b/parcels/roles/services/galaxy-report-service/templates/galaxy-report-service.j2 new file mode 100644 index 0000000..4063d34 --- /dev/null +++ b/parcels/roles/services/galaxy-report-service/templates/galaxy-report-service.j2 @@ -0,0 +1,123 @@ +#http的端口 +server: + port: 9093 +#更新进度条的时间10s +scan: + result: + scheduled: + plan: 0/15 * * * * ? +#同时间执行是线程数 +globle: + job_thread: 2 +#Hbasehttp的端口 +#Hbase的表名等配置通畅不需要更改 +hbase: + table: tsg:report_result + zookeeper_quorum: {{ zookeeper_servers }} + zookeeper_property_clientPort: 2181 + zookeeper_znode_parent: /hbase + client_retries_number: 3 + rpc_timeout: 100000 + connect_pool: 10 +#查询网关ip +ck: + gateway_ip: {{ gateway_keepalive_host }}:9999 +#zk集群的ip +zookeeper: + connectString: {{ zookeeper_servers }} +#是否启用zookeeper 0启用(集群) 1禁用(单机) + open: 0 + retryCount: 6 + elapsedTimeMs: 10000 + sessionTimeoutMs: 50000 + connectionTimeoutMs: 50000 + nameSpace: reportservice + +#最大连接数 +http: + maxTotal: 300 +#并发数 + defaultMaxPerRoute: 100 +#创建连接的最长时间 + connectTimeout: 10000 +#从连接池中获取到连接的最长时间 + connectionRequestTimeout: 10000 +#数据传输的最长时间 + socketTimeout: 21605000 +#提交请求前测试连接是否可用 + staleConnectionCheckEnabled: true + socketTimeoutShort: 30000 + +#mariadb的url +spring: + application: + name: galaxy_report_service + + datasource: + url: jdbc:mariadb://{{ bifang_mariadb_host }}:3306/tsg-bifang?serverTimezone=GMT&useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false +#mariadb的用户名 + username: root +#mariadb的密码 + password: {{ bifang_mariadb_pin }} + +#以下配置不需要更改通常 + name: druidDataSource + type: com.alibaba.druid.pool.DruidDataSource + driver-class-name: org.mariadb.jdbc.Driver + +#配置监控统计拦截的filters，去掉后监控界面SQL无法进行统计，’wall’用于防火墙 + druid: + filters: stat,wall,slf4j + #最大连接数 + max-active: 30 + #最小连接数 + min-idle: 1 + #初始化连接数 + initial-size: 2 + #获取连接最大超时时间 + max-wait: 600000 + #间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒 + time-between-eviction-runs-millis: 60000 + # 一个连接在池中最小生存的时间，单位是毫秒 + min-evictable-idle-time-millis: 300000 + #验证连接是否可用,在数据库中执行一条sql + validation-query: select 1 + #建议配置为true，不影响性能，并且保证安全性。申请连接的时候检测，如果空闲时间大于timeBetweenEvictionRunsMillis， + # 执行validationQuery检测连接是否有效 + test-while-idle: true + #申请连接时执行validationQuery检测连接是否有效，做了这个配置会降低性能 + test-on-borrow: true + test-on-return: false + connection-properties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=500 + #是否开启WebStatFilter + web-stat-filter: + enabled: true + #设置不统计哪些URL(用于排除一些不必要的url) + exclusions: "*.js,*.gif,*.jpg,*..ng,*.css,*.ico,/druid/*" + #是否开启Druid监控信息显示页面 + stat-view-servlet: + enabled: true + #甚至浏览器访问路径 + url-pattern: /druid/* + #禁止手动重置监控数据 + reset-enable: false + #durid-ui页面账户密码 + login-username: admin + login-password: admin + #Spring监控，对内部各接口调用的监控,需要导入aop相关包 + aop-patterns: com.mesa.reportservice.controller.*,com.mesa.reportservice.service.*,com.mesa.reportservice.mapper.* +mybatis: + typeAliasesPackage: com.mesa.reportservice.bean + mapperLocations: classpath*:/mappers/*.xml +management: + endpoints: + web: + exposure: + include: "*" + metrics: + tags: + application: galaxy_report_service + +logging: + config: ./config/log4j2-dev.xml + diff --git a/parcels/roles/services/galaxy-report-service/templates/tmp/application.properties.j2 b/parcels/roles/services/galaxy-report-service/templates/tmp/application.properties.j2 new file mode 100644 index 0000000..95042b9 --- /dev/null +++ b/parcels/roles/services/galaxy-report-service/templates/tmp/application.properties.j2 @@ -0,0 +1,100 @@ +#Hbasehttp的端口 +server.port=9093 +#更新进度条的时间10s +scan.result.scheduled.plan=0/10 * * * * ? +#同时间执行是线程数 +globle.job_thread=2 + +#Hbasehttp的端口 +#Hbase的表名等配置通畅不需要更改 +hbase.table=tsg:report_result +hbase.zookeeper_quorum={{ zookeeper_servers }} +hbase.zookeeper_property_clientPort=2181 +hbase.zookeeper_znode_parent=/hbase +hbase.client_retries_number=3 +hbase.rpc_timeout=100000 +hbase.connect_pool=10 + +#查询网关ip +ck.gateway_ip={{ gateway_keepalive_host }}:9999 +#mariadb的url +spring.datasource.url=jdbc:mariadb://{{ bifang_mariadb_host }}:3306/tsg-bifang?serverTimezone=GMT&useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false +#mariadb的用户名 +spring.datasource.username=root +#mariadb的密码 +spring.datasource.password={{ bifang_mariadb_pin }} + +#zk集群的ip +zookeeper.connectString={{ zookeeper_servers }} +#是否启用zookeeper 0启用(集群) 1禁用(单机) +zookeeper.open=0 +zookeeper.retryCount=5 +zookeeper.elapsedTimeMs=10000 +zookeeper.sessionTimeoutMs=50000 +zookeeper.connectionTimeoutMs=50000 +zookeeper.nameSpace=reportservice + +#最大连接数 +http.maxTotal=300 +#并发数 +http.defaultMaxPerRoute=100 +#创建连接的最长时间 +http.connectTimeout=10000 +#从连接池中获取到连接的最长时间 +http.connectionRequestTimeout=10000 +#数据传输的最长时间 +http.socketTimeout=21605000 +#提交请求前测试连接是否可用 +http.staleConnectionCheckEnabled=true +http.socketTimeoutShort=30000 + +#以下配置不需要更改通常 +spring.datasource.name=druidDataSource +spring.datasource.type=com.alibaba.druid.pool.DruidDataSource +spring.datasource.driver-class-name=org.mariadb.jdbc.Driver + +#配置监控统计拦截的filters，去掉后监控界面SQL无法进行统计，’wall’用于防火墙 +spring.datasource.druid.filters=stat,wall,slf4j +#最大连接数 +spring.datasource.druid.max-active=30 +#最小连接数 +spring.datasource.druid.min-idle=1 +#初始化连接数 +spring.datasource.druid.initial-size=2 +#获取连接最大超时时间 +spring.datasource.druid.max-wait=600000 +#间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒 +spring.datasource.druid.time-between-eviction-runs-millis=60000 +# 一个连接在池中最小生存的时间，单位是毫秒 +spring.datasource.druid.min-evictable-idle-time-millis=300000 +#验证连接是否可用,在数据库中执行一条sql +spring.datasource.druid.validation-query=select 1 +#建议配置为true，不影响性能，并且保证安全性。申请连接的时候检测，如果空闲时间大于timeBetweenEvictionRunsMillis， +# 执行validationQuery检测连接是否有效 +spring.datasource.druid.test-while-idle=true +#申请连接时执行validationQuery检测连接是否有效，做了这个配置会降低性能 +spring.datasource.druid.test-on-borrow=true +spring.datasource.druid.test-on-return=false +spring.datasource.druid.connection-properties=druid.stat.mergeSql=true;druid.stat.slowSqlMillis=500 +#是否开启WebStatFilter +spring.datasource.druid.web-stat-filter.enabled=true +#设置不统计哪些URL(用于排除一些不必要的url) +spring.datasource.druid.web-stat-filter.exclusions=*.js,*.gif,*.jpg,*.png,*.css,*.ico,/druid/* +#是否开启Druid监控信息显示页面 +spring.datasource.druid.stat-view-servlet.enabled=true +#甚至浏览器访问路径 +spring.datasource.druid.stat-view-servlet.url-pattern=/druid/* +#禁止手动重置监控数据 +spring.datasource.druid.stat-view-servlet.reset-enable=false +#durid-ui页面账户密码 +spring.datasource.druid.stat-view-servlet.login-username=admin +spring.datasource.druid.stat-view-servlet.login-password=admin +#Spring监控，对内部各接口调用的监控,需要导入aop相关包 +spring.datasource.druid.aop-patterns=com.mesa.reportservice.controller.*,com.mesa.reportservice.service.*,com.mesa.reportservice.mapper.* +mybatis.typeAliasesPackage=com.mesa.reportservice.bean +mybatis.mapperLocations=classpath*:/mappers/*.xml +spring.application.name=galaxy_reort_service +management.endpoints.web.exposure.include=* +management.metrics.tags.application=${spring.application.name} + +logging.config: ./config/log4j2-dev.xml diff --git a/parcels/roles/services/galaxy-report-service/vars/main.yml b/parcels/roles/services/galaxy-report-service/vars/main.yml new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/parcels/roles/services/galaxy-report-service/vars/main.yml diff --git a/parcels/test.yml b/parcels/test.yml new file mode 100644 index 0000000..d268252 --- /dev/null +++ b/parcels/test.yml @@ -0,0 +1,11 @@ + +- hosts: zookeeper + remote_user: root + roles: + - components/test +# vars_files: +# - ../configurations/config.yml +# - ../configurations/components.yml +# - ../configurations/services.yml +# - roles/components/zookeeper/vars/main.yml + diff --git a/parcels/test1.yml b/parcels/test1.yml new file mode 100644 index 0000000..b6f6a77 --- /dev/null +++ b/parcels/test1.yml @@ -0,0 +1,7 @@ +- hosts: 127.0.0.1 + remote_user: root + gather_facts: yes + tasks: + - name: message + debug: + msg: "{{ package_path }} {{ playbook_dir | dirname }} {{ qgw_image_tar_name }}" |