提交tsg-olap 24.01版本初始化剧本

170 files changed, 13137 insertions, 0 deletions

diff --git a/tsg-olap/parcels/roles/initialization/arangodb/tasks/check.yml b/tsg-olap/parcels/roles/initialization/arangodb/tasks/check.yml
new file mode 100644
index 0000000..dfb0b32
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/arangodb/tasks/check.yml
@@ -0,0 +1,11 @@
+- name: Set arangodb check variable
+  set_fact:
+     pin: "root:{{ arangodb_default_pin }}"
+
+- name: Check if collection exists in ArangoDB
+  uri:
+    url: "http://{{ groups.arangodb[0] }}:8529/_db/{{ arangodb_database }}/_api/collection/{{ item }}"
+    method: GET
+    headers:
+       Authorization: Basic {{ pin | b64encode }}
+  loop: "{{ arangodb_collections }}"
diff --git a/tsg-olap/parcels/roles/initialization/arangodb/tasks/init.yml b/tsg-olap/parcels/roles/initialization/arangodb/tasks/init.yml
new file mode 100644
index 0000000..920b6d4
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/arangodb/tasks/init.yml
@@ -0,0 +1,21 @@
+- name: Creating directory
+  file:
+    state: directory
+    path: '{{ deploy_dir }}/{{ container_name }}/init'
+
+- name: Copying init files
+  template:
+    src: "create_database.sh.j2"
+    dest: "{{ deploy_dir }}/{{ container_name }}/init/create_database.sh"
+    mode: 0755
+    force: true
+
+- name: Creating database and collection
+  shell: 'cd {{ deploy_dir }}/{{ container_name }}/init && sh create_database.sh'
+  register: response
+  until: response.rc == 0
+  retries: 3
+  delay: 3
+  failed_when: response.rc != 0
+  run_once: true
+
diff --git a/tsg-olap/parcels/roles/initialization/arangodb/tasks/main.yml b/tsg-olap/parcels/roles/initialization/arangodb/tasks/main.yml
new file mode 100644
index 0000000..8894d83
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/arangodb/tasks/main.yml
@@ -0,0 +1,3 @@
+- include: init.yml
+  when: operation == 'install'
+- include: check.yml
diff --git a/tsg-olap/parcels/roles/initialization/arangodb/templates/create_database.sh.j2 b/tsg-olap/parcels/roles/initialization/arangodb/templates/create_database.sh.j2
new file mode 100644
index 0000000..29657da
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/arangodb/templates/create_database.sh.j2
@@ -0,0 +1,54 @@
+#!/bin/bash
+source /etc/profile
+
+arangodb_host={{ groups.arangodb[0] | default('localhost') }}
+username=root
+pin="{{ arangodb_default_pin }}"
+database="{{ arangodb_database }}"
+
+function create_database() {
+	# 检查数据库是否存在
+        sleep 2
+	check_database=$(curl -u $username:$pin -s -o /dev/null -w "%{http_code}" http://$arangodb_host:8529/_db/$database/_api/database/current)
+	if [[ $check_database -eq 200 ]]; then
+		echo "Database $database already exists."
+	else
+		# 创建数据库
+		local response_code=$(curl -u $username:$pin -s -o /dev/null -w "%{http_code}" -X POST -H "accept: application/json" -d "{\"name\": \"$database\"}" http://$arangodb_host:8529/_api/database)
+		if [[ $response_code -eq 201 ]]; then
+			echo "Database $database created successfully."
+		else
+			echo "Failed to create database $database."
+			exit 1
+		fi
+	fi
+        sleep 2
+
+}
+
+function create_collection() {
+	local collection_name=$1
+	local collection_type=$2
+        sleep 2
+	# 检查文档是否存在
+	check_collection=$(curl -u $username:$pin -s -o /dev/null -w "%{http_code}" http://$arangodb_host:8529/_db/$database/_api/collection/$collection_name)
+	if [[ $check_collection -eq 200 ]]; then
+		echo "Collection $collection_name already exists."
+	else
+		# 创建文档$collection_name
+		local response_code=$(curl -u $username:$pin -s -o /dev/null -w "%{http_code}" -X POST -H 'accept: application/json' -d "{\"name\": \"$collection_name\",\"type\": $collection_type }" http://$arangodb_host:8529/_db/$database/_api/collection?isSystem=false)
+		if [[ $response_code -eq 201 ]]; then
+			echo "Collection $collection_name created successfully."
+		else
+			echo "Failed to create collection $collection_name."
+			exit 1
+		fi
+	fi
+        sleep 2
+}
+
+# 2: document 3: edges
+create_database
+create_collection IP 2
+create_collection R_LOCATE_FQDN2IP 3
+
diff --git a/tsg-olap/parcels/roles/initialization/arangodb/vars/main.yml b/tsg-olap/parcels/roles/initialization/arangodb/vars/main.yml
new file mode 100644
index 0000000..fa8b9d4
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/arangodb/vars/main.yml
@@ -0,0 +1,9 @@
+container_name: arangodb
+
+# 数据库
+arangodb_database: 'tsg_galaxy_v3'
+
+# 库表
+arangodb_collections:
+   - IP
+   - R_LOCATE_FQDN2IP
diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/create_ck_table.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/create_ck_table.sql
new file mode 100644
index 0000000..7eb3923
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/create_ck_table.sql
@@ -0,0 +1,4174 @@
+create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_cluster;
+create database IF NOT EXISTS tsg_galaxy_v3 ON CLUSTER ck_query;
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event_local on cluster ck_cluster (
+    vsys_id Int32,
+    recv_time Int64,
+    log_id UInt64,
+    profile_id Int64,
+    start_time Int64,
+    end_time Int64,
+    attack_type String,
+    severity String,
+    conditions String,
+    destination_ip String,
+    destination_country String,
+    source_ip_list String,
+    source_country_list String,
+    session_rate Int64,
+    packet_rate Int64,
+    bit_rate Int64
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(toDate(recv_time))
+ORDER BY (vsys_id,destination_ip,recv_time,log_id);
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_cluster (
+    vsys_id Int32,
+    recv_time Int64,
+    log_id UInt64,
+    profile_id Int64,
+    start_time Int64,
+    end_time Int64,
+    attack_type String,
+    severity String,
+    conditions String,
+    destination_ip String,
+    destination_country String,
+    source_ip_list String,
+    source_country_list String,
+    session_rate Int64,
+    packet_rate Int64,
+    bit_rate Int64
+)
+ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand());
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.dos_event on cluster ck_query (
+    vsys_id Int32,
+    recv_time Int64,
+    log_id UInt64,
+    profile_id Int64,
+    start_time Int64,
+    end_time Int64,
+    attack_type String,
+    severity String,
+    conditions String,
+    destination_ip String,
+    destination_country String,
+    source_ip_list String,
+    source_country_list String,
+    session_rate Int64,
+    packet_rate Int64,
+    bit_rate Int64
+)
+ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,dos_event_local,rand());
+
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event_local on cluster ck_cluster (
+        log_id UInt64,
+        recv_time Int64,
+        vsys_id Int64,
+        assessment_date Int64,
+        lot_number String,
+        file_name String,
+        assessment_file        String,
+        assessment_type        String,
+        features String,
+        size Int64,
+        file_checksum_sha String
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(toDate(recv_time))
+ORDER BY (vsys_id,recv_time,log_id);
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_query (
+        log_id UInt64,
+        recv_time Int64,
+        vsys_id Int64,
+        assessment_date Int64,
+        lot_number String,
+        file_name String,
+        assessment_file        String,
+        assessment_type        String,
+        features String,
+        size Int64,
+        file_checksum_sha String
+)
+ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand());
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.assessment_event on cluster ck_cluster (
+        log_id UInt64,
+        recv_time Int64,
+        vsys_id Int64,
+        assessment_date Int64,
+        lot_number String,
+        file_name String,
+        assessment_file        String,
+        assessment_type        String,
+        features String,
+        size Int64,
+        file_checksum_sha String
+)
+ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,assessment_event_local,rand());
+
+
+
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_local on cluster ck_cluster (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+start_timestamp_ms DateTime64(3),
+end_timestamp_ms DateTime64(3),
+duration_ms Int32,
+tcp_handshake_latency_ms Nullable(Int32),
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
+device_id String,
+out_link_id Nullable(Int32),
+in_link_id Nullable(Int32),
+device_tag String,
+data_center String,
+device_group String,
+sled_ip String,
+address_type Int32,
+vsys_id Int32,
+t_vsys_id Int32,
+flags Int64,
+flags_identify_info String,
+security_rule_list Array(Int64),
+security_action String,
+monitor_rule_list Array(Int64),
+shaping_rule_list Array(Int64),
+proxy_rule_list Array(Int64),
+statistics_rule_list Array(Int64),
+sc_rule_list Array(Int64),
+sc_rsp_raw Array(Int64),
+sc_rsp_decrypted Array(Int64),
+proxy_action String,
+proxy_pinning_status Nullable(Int32),
+proxy_intercept_status Nullable(Int32),
+proxy_passthrough_reason String,
+proxy_client_side_latency_ms Nullable(Int32),
+proxy_server_side_latency_ms Nullable(Int32),
+proxy_client_side_version String,
+proxy_server_side_version String,
+proxy_cert_verify Nullable(Int32),
+proxy_intercept_error String,
+monitor_mirrored_pkts Nullable(Int32),
+monitor_mirrored_bytes Nullable(Int32),
+client_ip String,
+client_port Int32,
+client_os_desc String,
+client_geolocation LowCardinality(String),
+client_asn Nullable(Int64),
+subscriber_id String,
+imei String,
+imsi String,
+phone_number String,
+apn String,
+server_ip String,
+server_port Int32,
+server_os_desc String,
+server_geolocation LowCardinality(String),
+server_asn Nullable(Int64),
+server_fqdn String,
+server_domain String,
+app_transition String, 
+app LowCardinality(String),
+app_debug_info String,
+app_content String,
+fqdn_category_list Array(Int64),
+ip_protocol LowCardinality(String),
+decoded_path LowCardinality(String),
+dns_message_id Nullable(Int32),
+dns_qr Nullable(Int32),
+dns_opcode Nullable(Int32),
+dns_aa Nullable(Int32),
+dns_tc Nullable(Int32),
+dns_rd Nullable(Int32),
+dns_ra Nullable(Int32),
+dns_rcode Nullable(Int32),
+dns_qdcount Nullable(Int32),
+dns_ancount Nullable(Int32),
+dns_nscount Nullable(Int32),
+dns_arcount Nullable(Int32),
+dns_qname String,
+dns_qtype Nullable(Int32),
+dns_qclass Nullable(Int32),
+dns_cname String,
+dns_sub Nullable(Int32),
+dns_rr String,
+dns_response_latency_ms Nullable(Int32),
+http_url String,
+http_host String,
+http_request_line String,
+http_response_line String,
+http_request_body String,
+http_response_body String,
+http_proxy_flag Nullable(Int32),
+http_sequence Nullable(Int32),
+http_cookie String,
+http_referer String,
+http_user_agent String,
+http_request_content_length Nullable(Int64),
+http_request_content_type String,
+http_response_content_length Nullable(Int64),
+http_response_content_type String,
+http_set_cookie String,
+http_version String,
+http_status_code Nullable(Int32),
+http_response_latency_ms Nullable(Int32),
+http_session_duration_ms Nullable(Int32),
+http_action_file_size Nullable(Int64),
+ssl_version String,
+ssl_sni String,
+ssl_san String,
+ssl_cn String,
+ssl_handshake_latency_ms Nullable(Int32),
+ssl_ja3_hash String,
+ssl_ja3s_hash String,
+ssl_cert_issuer String,
+ssl_cert_subject String,
+ssl_esni_flag Nullable(Int32),
+ssl_ech_flag Nullable(Int32),
+dtls_cookie String,
+dtls_version  String,
+dtls_sni String,
+dtls_san String,
+dtls_cn String,
+dtls_handshake_latency_ms Nullable(Int32),
+dtls_ja3_fingerprint String,
+dtls_ja3_hash String,
+dtls_cert_issuer String,
+dtls_cert_subject String,
+mail_protocol_type String,
+mail_account String,
+mail_from_cmd String,
+mail_to_cmd String,
+mail_from String,
+mail_password String,
+mail_to String,
+mail_cc String,
+mail_bcc String,
+mail_subject String,
+mail_subject_charset String,
+mail_attachment_name String,
+mail_attachment_name_charset String,
+mail_eml_file String,
+ftp_account String,
+ftp_url String,
+ftp_link_type String,
+quic_version String,
+quic_sni String,
+quic_user_agent String,
+rdp_cookie String,
+rdp_security_protocol String,
+rdp_client_channels String,
+rdp_keyboard_layout String,
+rdp_client_version String,
+rdp_client_name String,
+rdp_client_product_id String,
+rdp_desktop_width String,
+rdp_desktop_height String,
+rdp_requested_color_depth String,
+rdp_certificate_type String,
+rdp_certificate_count Nullable(Int32),
+rdp_certificate_permanent Nullable(Int32),
+rdp_encryption_level String,
+rdp_encryption_method String,
+ssh_version String,
+ssh_auth_success String,
+ssh_client_version String,
+ssh_server_version String,
+ssh_cipher_alg String,
+ssh_mac_alg String,
+ssh_compression_alg String,
+ssh_kex_alg String,
+ssh_host_key_alg String,
+ssh_host_key String,
+ssh_hassh String,
+sip_call_id String,
+sip_originator_description String,
+sip_responder_description String,
+sip_user_agent String,
+sip_server String,
+sip_originator_sdp_connect_ip String,
+sip_originator_sdp_media_port Nullable(Int32),
+sip_originator_sdp_media_type String,
+sip_originator_sdp_content String,
+sip_responder_sdp_connect_ip String,
+sip_responder_sdp_media_port Nullable(Int32),
+sip_responder_sdp_media_type String,
+sip_responder_sdp_content String,
+sip_duration_s Nullable(Int32),
+sip_bye String,
+rtp_payload_type_c2s Nullable(Int32),
+rtp_payload_type_s2c Nullable(Int32),
+rtp_pcap_path String,
+rtp_originator_dir Nullable(Int32),
+stratum_cryptocurrency String,
+stratum_mining_pools String,
+stratum_mining_program String,
+stratum_mining_subscribe String,
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+tcp_c2s_ip_fragments Nullable(Int64),
+tcp_s2c_ip_fragments Nullable(Int64),
+tcp_c2s_lost_bytes Nullable(Int64),
+tcp_s2c_lost_bytes Nullable(Int64),
+tcp_c2s_o3_pkts Nullable(Int64),
+tcp_s2c_o3_pkts Nullable(Int64),
+tcp_c2s_rtx_pkts Nullable(Int64),
+tcp_s2c_rtx_pkts Nullable(Int64),
+tcp_c2s_rtx_bytes Nullable(Int64),
+tcp_s2c_rtx_bytes Nullable(Int64),
+tcp_rtt_ms Nullable(Int32),
+tcp_client_isn Nullable(Int64),
+tcp_server_isn Nullable(Int64),
+packet_capture_file String,
+in_src_mac String,
+out_src_mac String,
+in_dest_mac String,
+out_dest_mac String,
+tunnels String,
+dup_traffic_flag Nullable(Int32),
+tunnel_endpoint_a_desc String,
+tunnel_endpoint_b_desc String
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(toDate(recv_time))
+ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time);
+
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record on cluster ck_cluster (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+start_timestamp_ms DateTime64(3),
+end_timestamp_ms DateTime64(3),
+duration_ms Int32,
+tcp_handshake_latency_ms Nullable(Int32),
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64,
+device_id String,
+out_link_id Nullable(Int32),
+in_link_id Nullable(Int32),
+device_tag String,
+data_center String,
+device_group String,
+sled_ip String,
+address_type Int32,
+vsys_id Int32,
+t_vsys_id Int32,
+flags Int64,
+flags_identify_info String,
+security_rule_list Array(Int64),
+security_action String,
+monitor_rule_list Array(Int64),
+shaping_rule_list Array(Int64),
+proxy_rule_list Array(Int64),
+statistics_rule_list Array(Int64),
+sc_rule_list Array(Int64),
+sc_rsp_raw Array(Int64),
+sc_rsp_decrypted Array(Int64),
+proxy_action String,
+proxy_pinning_status Nullable(Int32),
+proxy_intercept_status Nullable(Int32),
+proxy_passthrough_reason String,
+proxy_client_side_latency_ms Nullable(Int32),
+proxy_server_side_latency_ms Nullable(Int32),
+proxy_client_side_version String,
+proxy_server_side_version String,
+proxy_cert_verify Nullable(Int32),
+proxy_intercept_error String,
+monitor_mirrored_pkts Nullable(Int32),
+monitor_mirrored_bytes Nullable(Int32),
+client_ip String,
+client_port Int32,
+client_os_desc String,
+client_geolocation LowCardinality(String),
+client_asn Nullable(Int64),
+subscriber_id String,
+imei String,
+imsi String,
+phone_number String,
+apn String,
+server_ip String,
+server_port Int32,
+server_os_desc String,
+server_geolocation LowCardinality(String),
+server_asn Nullable(Int64),
+server_fqdn String,
+server_domain String,
+app_transition String, 
+app LowCardinality(String),
+app_debug_info String,
+app_content String,
+fqdn_category_list Array(Int64),
+ip_protocol LowCardinality(String),
+decoded_path LowCardinality(String),
+dns_message_id Nullable(Int32),
+dns_qr Nullable(Int32),
+dns_opcode Nullable(Int32),
+dns_aa Nullable(Int32),
+dns_tc Nullable(Int32),
+dns_rd Nullable(Int32),
+dns_ra Nullable(Int32),
+dns_rcode Nullable(Int32),
+dns_qdcount Nullable(Int32),
+dns_ancount Nullable(Int32),
+dns_nscount Nullable(Int32),
+dns_arcount Nullable(Int32),
+dns_qname String,
+dns_qtype Nullable(Int32),
+dns_qclass Nullable(Int32),
+dns_cname String,
+dns_sub Nullable(Int32),
+dns_rr String,
+dns_response_latency_ms Nullable(Int32),
+http_url String,
+http_host String,
+http_request_line String,
+http_response_line String,
+http_request_body String,
+http_response_body String,
+http_proxy_flag Nullable(Int32),
+http_sequence Nullable(Int32),
+http_cookie String,
+http_referer String,
+http_user_agent String,
+http_request_content_length Nullable(Int64),
+http_request_content_type String,
+http_response_content_length Nullable(Int64),
+http_response_content_type String,
+http_set_cookie String,
+http_version String,
+http_status_code Nullable(Int32),
+http_response_latency_ms Nullable(Int32),
+http_session_duration_ms Nullable(Int32),
+http_action_file_size Nullable(Int64),
+ssl_version String,
+ssl_sni String,
+ssl_san String,
+ssl_cn String,
+ssl_handshake_latency_ms Nullable(Int32),
+ssl_ja3_hash String,
+ssl_ja3s_hash String,
+ssl_cert_issuer String,
+ssl_cert_subject String,
+ssl_esni_flag Nullable(Int32),
+ssl_ech_flag Nullable(Int32),
+dtls_cookie String,
+dtls_version  String,
+dtls_sni String,
+dtls_san String,
+dtls_cn String,
+dtls_handshake_latency_ms Nullable(Int32),
+dtls_ja3_fingerprint String,
+dtls_ja3_hash String,
+dtls_cert_issuer String,
+dtls_cert_subject String,
+mail_protocol_type String,
+mail_account String,
+mail_from_cmd String,
+mail_to_cmd String,
+mail_from String,
+mail_password String,
+mail_to String,
+mail_cc String,
+mail_bcc String,
+mail_subject String,
+mail_subject_charset String,
+mail_attachment_name String,
+mail_attachment_name_charset String,
+mail_eml_file String,
+ftp_account String,
+ftp_url String,
+ftp_link_type String,
+quic_version String,
+quic_sni String,
+quic_user_agent String,
+rdp_cookie String,
+rdp_security_protocol String,
+rdp_client_channels String,
+rdp_keyboard_layout String,
+rdp_client_version String,
+rdp_client_name String,
+rdp_client_product_id String,
+rdp_desktop_width String,
+rdp_desktop_height String,
+rdp_requested_color_depth String,
+rdp_certificate_type String,
+rdp_certificate_count Nullable(Int32),
+rdp_certificate_permanent Nullable(Int32),
+rdp_encryption_level String,
+rdp_encryption_method String,
+ssh_version String,
+ssh_auth_success String,
+ssh_client_version String,
+ssh_server_version String,
+ssh_cipher_alg String,
+ssh_mac_alg String,
+ssh_compression_alg String,
+ssh_kex_alg String,
+ssh_host_key_alg String,
+ssh_host_key String,
+ssh_hassh String,
+sip_call_id String,
+sip_originator_description String,
+sip_responder_description String,
+sip_user_agent String,
+sip_server String,
+sip_originator_sdp_connect_ip String,
+sip_originator_sdp_media_port Nullable(Int32),
+sip_originator_sdp_media_type String,
+sip_originator_sdp_content String,
+sip_responder_sdp_connect_ip String,
+sip_responder_sdp_media_port Nullable(Int32),
+sip_responder_sdp_media_type String,
+sip_responder_sdp_content String,
+sip_duration_s Nullable(Int32),
+sip_bye String,
+rtp_payload_type_c2s Nullable(Int32),
+rtp_payload_type_s2c Nullable(Int32),
+rtp_pcap_path String,
+rtp_originator_dir Nullable(Int32),
+stratum_cryptocurrency String,
+stratum_mining_pools String,
+stratum_mining_program String,
+stratum_mining_subscribe String,
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+tcp_c2s_ip_fragments Nullable(Int64),
+tcp_s2c_ip_fragments Nullable(Int64),
+tcp_c2s_lost_bytes Nullable(Int64),
+tcp_s2c_lost_bytes Nullable(Int64),
+tcp_c2s_o3_pkts Nullable(Int64),
+tcp_s2c_o3_pkts Nullable(Int64),
+tcp_c2s_rtx_pkts Nullable(Int64),
+tcp_s2c_rtx_pkts Nullable(Int64),
+tcp_c2s_rtx_bytes Nullable(Int64),
+tcp_s2c_rtx_bytes Nullable(Int64),
+tcp_rtt_ms Nullable(Int32),
+tcp_client_isn Nullable(Int64),
+tcp_server_isn Nullable(Int64),
+packet_capture_file String,
+in_src_mac String,
+out_src_mac String,
+in_dest_mac String,
+out_dest_mac String,
+tunnels String,
+dup_traffic_flag Nullable(Int32),
+tunnel_endpoint_a_desc String,
+tunnel_endpoint_b_desc String
+)
+ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand());
+
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record on cluster ck_query (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+start_timestamp_ms DateTime64(3),
+end_timestamp_ms DateTime64(3),
+duration_ms Int32,
+tcp_handshake_latency_ms Nullable(Int32),
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64,
+device_id String,
+out_link_id Nullable(Int32),
+in_link_id Nullable(Int32),
+device_tag String,
+data_center String,
+device_group String,
+sled_ip String,
+address_type Int32,
+vsys_id Int32,
+t_vsys_id Int32,
+flags Int64,
+flags_identify_info String,
+security_rule_list Array(Int64),
+security_action String,
+monitor_rule_list Array(Int64),
+shaping_rule_list Array(Int64),
+proxy_rule_list Array(Int64),
+statistics_rule_list Array(Int64),
+sc_rule_list Array(Int64),
+sc_rsp_raw Array(Int64),
+sc_rsp_decrypted Array(Int64),
+proxy_action String,
+proxy_pinning_status Nullable(Int32),
+proxy_intercept_status Nullable(Int32),
+proxy_passthrough_reason String,
+proxy_client_side_latency_ms Nullable(Int32),
+proxy_server_side_latency_ms Nullable(Int32),
+proxy_client_side_version String,
+proxy_server_side_version String,
+proxy_cert_verify Nullable(Int32),
+proxy_intercept_error String,
+monitor_mirrored_pkts Nullable(Int32),
+monitor_mirrored_bytes Nullable(Int32),
+client_ip String,
+client_port Int32,
+client_os_desc String,
+client_geolocation LowCardinality(String),
+client_asn Nullable(Int64),
+subscriber_id String,
+imei String,
+imsi String,
+phone_number String,
+apn String,
+server_ip String,
+server_port Int32,
+server_os_desc String,
+server_geolocation LowCardinality(String),
+server_asn Nullable(Int64),
+server_fqdn String,
+server_domain String,
+app_transition String, 
+app LowCardinality(String),
+app_debug_info String,
+app_content String,
+fqdn_category_list Array(Int64),
+ip_protocol LowCardinality(String),
+decoded_path LowCardinality(String),
+dns_message_id Nullable(Int32),
+dns_qr Nullable(Int32),
+dns_opcode Nullable(Int32),
+dns_aa Nullable(Int32),
+dns_tc Nullable(Int32),
+dns_rd Nullable(Int32),
+dns_ra Nullable(Int32),
+dns_rcode Nullable(Int32),
+dns_qdcount Nullable(Int32),
+dns_ancount Nullable(Int32),
+dns_nscount Nullable(Int32),
+dns_arcount Nullable(Int32),
+dns_qname String,
+dns_qtype Nullable(Int32),
+dns_qclass Nullable(Int32),
+dns_cname String,
+dns_sub Nullable(Int32),
+dns_rr String,
+dns_response_latency_ms Nullable(Int32),
+http_url String,
+http_host String,
+http_request_line String,
+http_response_line String,
+http_request_body String,
+http_response_body String,
+http_proxy_flag Nullable(Int32),
+http_sequence Nullable(Int32),
+http_cookie String,
+http_referer String,
+http_user_agent String,
+http_request_content_length Nullable(Int64),
+http_request_content_type String,
+http_response_content_length Nullable(Int64),
+http_response_content_type String,
+http_set_cookie String,
+http_version String,
+http_status_code Nullable(Int32),
+http_response_latency_ms Nullable(Int32),
+http_session_duration_ms Nullable(Int32),
+http_action_file_size Nullable(Int64),
+ssl_version String,
+ssl_sni String,
+ssl_san String,
+ssl_cn String,
+ssl_handshake_latency_ms Nullable(Int32),
+ssl_ja3_hash String,
+ssl_ja3s_hash String,
+ssl_cert_issuer String,
+ssl_cert_subject String,
+ssl_esni_flag Nullable(Int32),
+ssl_ech_flag Nullable(Int32),
+dtls_cookie String,
+dtls_version  String,
+dtls_sni String,
+dtls_san String,
+dtls_cn String,
+dtls_handshake_latency_ms Nullable(Int32),
+dtls_ja3_fingerprint String,
+dtls_ja3_hash String,
+dtls_cert_issuer String,
+dtls_cert_subject String,
+mail_protocol_type String,
+mail_account String,
+mail_from_cmd String,
+mail_to_cmd String,
+mail_from String,
+mail_password String,
+mail_to String,
+mail_cc String,
+mail_bcc String,
+mail_subject String,
+mail_subject_charset String,
+mail_attachment_name String,
+mail_attachment_name_charset String,
+mail_eml_file String,
+ftp_account String,
+ftp_url String,
+ftp_link_type String,
+quic_version String,
+quic_sni String,
+quic_user_agent String,
+rdp_cookie String,
+rdp_security_protocol String,
+rdp_client_channels String,
+rdp_keyboard_layout String,
+rdp_client_version String,
+rdp_client_name String,
+rdp_client_product_id String,
+rdp_desktop_width String,
+rdp_desktop_height String,
+rdp_requested_color_depth String,
+rdp_certificate_type String,
+rdp_certificate_count Nullable(Int32),
+rdp_certificate_permanent Nullable(Int32),
+rdp_encryption_level String,
+rdp_encryption_method String,
+ssh_version String,
+ssh_auth_success String,
+ssh_client_version String,
+ssh_server_version String,
+ssh_cipher_alg String,
+ssh_mac_alg String,
+ssh_compression_alg String,
+ssh_kex_alg String,
+ssh_host_key_alg String,
+ssh_host_key String,
+ssh_hassh String,
+sip_call_id String,
+sip_originator_description String,
+sip_responder_description String,
+sip_user_agent String,
+sip_server String,
+sip_originator_sdp_connect_ip String,
+sip_originator_sdp_media_port Nullable(Int32),
+sip_originator_sdp_media_type String,
+sip_originator_sdp_content String,
+sip_responder_sdp_connect_ip String,
+sip_responder_sdp_media_port Nullable(Int32),
+sip_responder_sdp_media_type String,
+sip_responder_sdp_content String,
+sip_duration_s Nullable(Int32),
+sip_bye String,
+rtp_payload_type_c2s Nullable(Int32),
+rtp_payload_type_s2c Nullable(Int32),
+rtp_pcap_path String,
+rtp_originator_dir Nullable(Int32),
+stratum_cryptocurrency String,
+stratum_mining_pools String,
+stratum_mining_program String,
+stratum_mining_subscribe String,
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+tcp_c2s_ip_fragments Nullable(Int64),
+tcp_s2c_ip_fragments Nullable(Int64),
+tcp_c2s_lost_bytes Nullable(Int64),
+tcp_s2c_lost_bytes Nullable(Int64),
+tcp_c2s_o3_pkts Nullable(Int64),
+tcp_s2c_o3_pkts Nullable(Int64),
+tcp_c2s_rtx_pkts Nullable(Int64),
+tcp_s2c_rtx_pkts Nullable(Int64),
+tcp_c2s_rtx_bytes Nullable(Int64),
+tcp_s2c_rtx_bytes Nullable(Int64),
+tcp_rtt_ms Nullable(Int32),
+tcp_client_isn Nullable(Int64),
+tcp_server_isn Nullable(Int64),
+packet_capture_file String,
+in_src_mac String,
+out_src_mac String,
+in_dest_mac String,
+out_dest_mac String,
+tunnels String,
+dup_traffic_flag Nullable(Int32),
+tunnel_endpoint_a_desc String,
+tunnel_endpoint_b_desc String
+)
+ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_local,rand());
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event_local on cluster ck_cluster (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+start_timestamp_ms DateTime64(3),
+end_timestamp_ms DateTime64(3),
+duration_ms Int32,
+tcp_handshake_latency_ms Nullable(Int32),
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
+device_id String,
+out_link_id Nullable(Int32),
+in_link_id Nullable(Int32),
+device_tag String,
+data_center String,
+device_group String,
+sled_ip String,
+address_type Int32,
+vsys_id Int32,
+t_vsys_id Int32,
+flags Int64,
+flags_identify_info String,
+security_rule_list Array(Int64),
+security_action String,
+monitor_rule_list Array(Int64),
+shaping_rule_list Array(Int64),
+proxy_rule_list Array(Int64),
+statistics_rule_list Array(Int64),
+sc_rule_list Array(Int64),
+sc_rsp_raw Array(Int64),
+sc_rsp_decrypted Array(Int64),
+proxy_action String,
+proxy_pinning_status Nullable(Int32),
+proxy_intercept_status Nullable(Int32),
+proxy_passthrough_reason String,
+proxy_client_side_latency_ms Nullable(Int32),
+proxy_server_side_latency_ms Nullable(Int32),
+proxy_client_side_version String,
+proxy_server_side_version String,
+proxy_cert_verify Nullable(Int32),
+proxy_intercept_error String,
+monitor_mirrored_pkts Nullable(Int32),
+monitor_mirrored_bytes Nullable(Int32),
+client_ip String,
+client_port Int32,
+client_os_desc String,
+client_geolocation LowCardinality(String),
+client_asn Nullable(Int64),
+subscriber_id String,
+imei String,
+imsi String,
+phone_number String,
+apn String,
+server_ip String,
+server_port Int32,
+server_os_desc String,
+server_geolocation LowCardinality(String),
+server_asn Nullable(Int64),
+server_fqdn String,
+server_domain String,
+app_transition String, 
+app LowCardinality(String),
+app_debug_info String,
+app_content String,
+fqdn_category_list Array(Int64),
+ip_protocol LowCardinality(String),
+decoded_path LowCardinality(String),
+dns_message_id Nullable(Int32),
+dns_qr Nullable(Int32),
+dns_opcode Nullable(Int32),
+dns_aa Nullable(Int32),
+dns_tc Nullable(Int32),
+dns_rd Nullable(Int32),
+dns_ra Nullable(Int32),
+dns_rcode Nullable(Int32),
+dns_qdcount Nullable(Int32),
+dns_ancount Nullable(Int32),
+dns_nscount Nullable(Int32),
+dns_arcount Nullable(Int32),
+dns_qname String,
+dns_qtype Nullable(Int32),
+dns_qclass Nullable(Int32),
+dns_cname String,
+dns_sub Nullable(Int32),
+dns_rr String,
+dns_response_latency_ms Nullable(Int32),
+http_url String,
+http_host String,
+http_request_line String,
+http_response_line String,
+http_request_body String,
+http_response_body String,
+http_proxy_flag Nullable(Int32),
+http_sequence Nullable(Int32),
+http_cookie String,
+http_referer String,
+http_user_agent String,
+http_request_content_length Nullable(Int64),
+http_request_content_type String,
+http_response_content_length Nullable(Int64),
+http_response_content_type String,
+http_set_cookie String,
+http_version String,
+http_status_code Nullable(Int32),
+http_response_latency_ms Nullable(Int32),
+http_session_duration_ms Nullable(Int32),
+http_action_file_size Nullable(Int64),
+ssl_version String,
+ssl_sni String,
+ssl_san String,
+ssl_cn String,
+ssl_handshake_latency_ms Nullable(Int32),
+ssl_ja3_hash String,
+ssl_ja3s_hash String,
+ssl_cert_issuer String,
+ssl_cert_subject String,
+ssl_esni_flag Nullable(Int32),
+ssl_ech_flag Nullable(Int32),
+dtls_cookie String,
+dtls_version  String,
+dtls_sni String,
+dtls_san String,
+dtls_cn String,
+dtls_handshake_latency_ms Nullable(Int32),
+dtls_ja3_fingerprint String,
+dtls_ja3_hash String,
+dtls_cert_issuer String,
+dtls_cert_subject String,
+mail_protocol_type String,
+mail_account String,
+mail_from_cmd String,
+mail_to_cmd String,
+mail_from String,
+mail_password String,
+mail_to String,
+mail_cc String,
+mail_bcc String,
+mail_subject String,
+mail_subject_charset String,
+mail_attachment_name String,
+mail_attachment_name_charset String,
+mail_eml_file String,
+ftp_account String,
+ftp_url String,
+ftp_link_type String,
+quic_version String,
+quic_sni String,
+quic_user_agent String,
+rdp_cookie String,
+rdp_security_protocol String,
+rdp_client_channels String,
+rdp_keyboard_layout String,
+rdp_client_version String,
+rdp_client_name String,
+rdp_client_product_id String,
+rdp_desktop_width String,
+rdp_desktop_height String,
+rdp_requested_color_depth String,
+rdp_certificate_type String,
+rdp_certificate_count Nullable(Int32),
+rdp_certificate_permanent Nullable(Int32),
+rdp_encryption_level String,
+rdp_encryption_method String,
+ssh_version String,
+ssh_auth_success String,
+ssh_client_version String,
+ssh_server_version String,
+ssh_cipher_alg String,
+ssh_mac_alg String,
+ssh_compression_alg String,
+ssh_kex_alg String,
+ssh_host_key_alg String,
+ssh_host_key String,
+ssh_hassh String,
+sip_call_id String,
+sip_originator_description String,
+sip_responder_description String,
+sip_user_agent String,
+sip_server String,
+sip_originator_sdp_connect_ip String,
+sip_originator_sdp_media_port Nullable(Int32),
+sip_originator_sdp_media_type String,
+sip_originator_sdp_content String,
+sip_responder_sdp_connect_ip String,
+sip_responder_sdp_media_port Nullable(Int32),
+sip_responder_sdp_media_type String,
+sip_responder_sdp_content String,
+sip_duration_s Nullable(Int32),
+sip_bye String,
+rtp_payload_type_c2s Nullable(Int32),
+rtp_payload_type_s2c Nullable(Int32),
+rtp_pcap_path String,
+rtp_originator_dir Nullable(Int32),
+stratum_cryptocurrency String,
+stratum_mining_pools String,
+stratum_mining_program String,
+stratum_mining_subscribe String,
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+tcp_c2s_ip_fragments Nullable(Int64),
+tcp_s2c_ip_fragments Nullable(Int64),
+tcp_c2s_lost_bytes Nullable(Int64),
+tcp_s2c_lost_bytes Nullable(Int64),
+tcp_c2s_o3_pkts Nullable(Int64),
+tcp_s2c_o3_pkts Nullable(Int64),
+tcp_c2s_rtx_pkts Nullable(Int64),
+tcp_s2c_rtx_pkts Nullable(Int64),
+tcp_c2s_rtx_bytes Nullable(Int64),
+tcp_s2c_rtx_bytes Nullable(Int64),
+tcp_rtt_ms Nullable(Int32),
+tcp_client_isn Nullable(Int64),
+tcp_server_isn Nullable(Int64),
+packet_capture_file String,
+in_src_mac String,
+out_src_mac String,
+in_dest_mac String,
+out_dest_mac String,
+tunnels String,
+dup_traffic_flag Nullable(Int32),
+tunnel_endpoint_a_desc String,
+tunnel_endpoint_b_desc String
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(toDate(recv_time))
+ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time);
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event on cluster ck_cluster (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+start_timestamp_ms DateTime64(3),
+end_timestamp_ms DateTime64(3),
+duration_ms Int32,
+tcp_handshake_latency_ms Nullable(Int32),
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64 ,
+device_id String,
+out_link_id Nullable(Int32),
+in_link_id Nullable(Int32),
+device_tag String,
+data_center String,
+device_group String,
+sled_ip String,
+address_type Int32,
+vsys_id Int32,
+t_vsys_id Int32,
+flags Int64,
+flags_identify_info String,
+security_rule_list Array(Int64),
+security_action String,
+monitor_rule_list Array(Int64),
+shaping_rule_list Array(Int64),
+proxy_rule_list Array(Int64),
+statistics_rule_list Array(Int64),
+sc_rule_list Array(Int64),
+sc_rsp_raw Array(Int64),
+sc_rsp_decrypted Array(Int64),
+proxy_action String,
+proxy_pinning_status Nullable(Int32),
+proxy_intercept_status Nullable(Int32),
+proxy_passthrough_reason String,
+proxy_client_side_latency_ms Nullable(Int32),
+proxy_server_side_latency_ms Nullable(Int32),
+proxy_client_side_version String,
+proxy_server_side_version String,
+proxy_cert_verify Nullable(Int32),
+proxy_intercept_error String,
+monitor_mirrored_pkts Nullable(Int32),
+monitor_mirrored_bytes Nullable(Int32),
+client_ip String,
+client_port Int32,
+client_os_desc String,
+client_geolocation LowCardinality(String),
+client_asn Nullable(Int64),
+subscriber_id String,
+imei String,
+imsi String,
+phone_number String,
+apn String,
+server_ip String,
+server_port Int32,
+server_os_desc String,
+server_geolocation LowCardinality(String),
+server_asn Nullable(Int64),
+server_fqdn String,
+server_domain String,
+app_transition String, 
+app LowCardinality(String),
+app_debug_info String,
+app_content String,
+fqdn_category_list Array(Int64),
+ip_protocol LowCardinality(String),
+decoded_path LowCardinality(String),
+dns_message_id Nullable(Int32),
+dns_qr Nullable(Int32),
+dns_opcode Nullable(Int32),
+dns_aa Nullable(Int32),
+dns_tc Nullable(Int32),
+dns_rd Nullable(Int32),
+dns_ra Nullable(Int32),
+dns_rcode Nullable(Int32),
+dns_qdcount Nullable(Int32),
+dns_ancount Nullable(Int32),
+dns_nscount Nullable(Int32),
+dns_arcount Nullable(Int32),
+dns_qname String,
+dns_qtype Nullable(Int32),
+dns_qclass Nullable(Int32),
+dns_cname String,
+dns_sub Nullable(Int32),
+dns_rr String,
+dns_response_latency_ms Nullable(Int32),
+http_url String,
+http_host String,
+http_request_line String,
+http_response_line String,
+http_request_body String,
+http_response_body String,
+http_proxy_flag Nullable(Int32),
+http_sequence Nullable(Int32),
+http_cookie String,
+http_referer String,
+http_user_agent String,
+http_request_content_length Nullable(Int64),
+http_request_content_type String,
+http_response_content_length Nullable(Int64),
+http_response_content_type String,
+http_set_cookie String,
+http_version String,
+http_status_code Nullable(Int32),
+http_response_latency_ms Nullable(Int32),
+http_session_duration_ms Nullable(Int32),
+http_action_file_size Nullable(Int64),
+ssl_version String,
+ssl_sni String,
+ssl_san String,
+ssl_cn String,
+ssl_handshake_latency_ms Nullable(Int32),
+ssl_ja3_hash String,
+ssl_ja3s_hash String,
+ssl_cert_issuer String,
+ssl_cert_subject String,
+ssl_esni_flag Nullable(Int32),
+ssl_ech_flag Nullable(Int32),
+dtls_cookie String,
+dtls_version  String,
+dtls_sni String,
+dtls_san String,
+dtls_cn String,
+dtls_handshake_latency_ms Nullable(Int32),
+dtls_ja3_fingerprint String,
+dtls_ja3_hash String,
+dtls_cert_issuer String,
+dtls_cert_subject String,
+mail_protocol_type String,
+mail_account String,
+mail_from_cmd String,
+mail_to_cmd String,
+mail_from String,
+mail_password String,
+mail_to String,
+mail_cc String,
+mail_bcc String,
+mail_subject String,
+mail_subject_charset String,
+mail_attachment_name String,
+mail_attachment_name_charset String,
+mail_eml_file String,
+ftp_account String,
+ftp_url String,
+ftp_link_type String,
+quic_version String,
+quic_sni String,
+quic_user_agent String,
+rdp_cookie String,
+rdp_security_protocol String,
+rdp_client_channels String,
+rdp_keyboard_layout String,
+rdp_client_version String,
+rdp_client_name String,
+rdp_client_product_id String,
+rdp_desktop_width String,
+rdp_desktop_height String,
+rdp_requested_color_depth String,
+rdp_certificate_type String,
+rdp_certificate_count Nullable(Int32),
+rdp_certificate_permanent Nullable(Int32),
+rdp_encryption_level String,
+rdp_encryption_method String,
+ssh_version String,
+ssh_auth_success String,
+ssh_client_version String,
+ssh_server_version String,
+ssh_cipher_alg String,
+ssh_mac_alg String,
+ssh_compression_alg String,
+ssh_kex_alg String,
+ssh_host_key_alg String,
+ssh_host_key String,
+ssh_hassh String,
+sip_call_id String,
+sip_originator_description String,
+sip_responder_description String,
+sip_user_agent String,
+sip_server String,
+sip_originator_sdp_connect_ip String,
+sip_originator_sdp_media_port Nullable(Int32),
+sip_originator_sdp_media_type String,
+sip_originator_sdp_content String,
+sip_responder_sdp_connect_ip String,
+sip_responder_sdp_media_port Nullable(Int32),
+sip_responder_sdp_media_type String,
+sip_responder_sdp_content String,
+sip_duration_s Nullable(Int32),
+sip_bye String,
+rtp_payload_type_c2s Nullable(Int32),
+rtp_payload_type_s2c Nullable(Int32),
+rtp_pcap_path String,
+rtp_originator_dir Nullable(Int32),
+stratum_cryptocurrency String,
+stratum_mining_pools String,
+stratum_mining_program String,
+stratum_mining_subscribe String,
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+tcp_c2s_ip_fragments Nullable(Int64),
+tcp_s2c_ip_fragments Nullable(Int64),
+tcp_c2s_lost_bytes Nullable(Int64),
+tcp_s2c_lost_bytes Nullable(Int64),
+tcp_c2s_o3_pkts Nullable(Int64),
+tcp_s2c_o3_pkts Nullable(Int64),
+tcp_c2s_rtx_pkts Nullable(Int64),
+tcp_s2c_rtx_pkts Nullable(Int64),
+tcp_c2s_rtx_bytes Nullable(Int64),
+tcp_s2c_rtx_bytes Nullable(Int64),
+tcp_rtt_ms Nullable(Int32),
+tcp_client_isn Nullable(Int64),
+tcp_server_isn Nullable(Int64),
+packet_capture_file String,
+in_src_mac String,
+out_src_mac String,
+in_dest_mac String,
+out_dest_mac String,
+tunnels String,
+dup_traffic_flag Nullable(Int32),
+tunnel_endpoint_a_desc String,
+tunnel_endpoint_b_desc String
+)
+ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand());
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.security_event on cluster ck_query (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+start_timestamp_ms DateTime64(3),
+end_timestamp_ms DateTime64(3),
+duration_ms Int32,
+tcp_handshake_latency_ms Nullable(Int32),
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64 ,
+device_id String,
+out_link_id Nullable(Int32),
+in_link_id Nullable(Int32),
+device_tag String,
+data_center String,
+device_group String,
+sled_ip String,
+address_type Int32,
+vsys_id Int32,
+t_vsys_id Int32,
+flags Int64,
+flags_identify_info String,
+security_rule_list Array(Int64),
+security_action String,
+monitor_rule_list Array(Int64),
+shaping_rule_list Array(Int64),
+proxy_rule_list Array(Int64),
+statistics_rule_list Array(Int64),
+sc_rule_list Array(Int64),
+sc_rsp_raw Array(Int64),
+sc_rsp_decrypted Array(Int64),
+proxy_action String,
+proxy_pinning_status Nullable(Int32),
+proxy_intercept_status Nullable(Int32),
+proxy_passthrough_reason String,
+proxy_client_side_latency_ms Nullable(Int32),
+proxy_server_side_latency_ms Nullable(Int32),
+proxy_client_side_version String,
+proxy_server_side_version String,
+proxy_cert_verify Nullable(Int32),
+proxy_intercept_error String,
+monitor_mirrored_pkts Nullable(Int32),
+monitor_mirrored_bytes Nullable(Int32),
+client_ip String,
+client_port Int32,
+client_os_desc String,
+client_geolocation LowCardinality(String),
+client_asn Nullable(Int64),
+subscriber_id String,
+imei String,
+imsi String,
+phone_number String,
+apn String,
+server_ip String,
+server_port Int32,
+server_os_desc String,
+server_geolocation LowCardinality(String),
+server_asn Nullable(Int64),
+server_fqdn String,
+server_domain String,
+app_transition String, 
+app LowCardinality(String),
+app_debug_info String,
+app_content String,
+fqdn_category_list Array(Int64),
+ip_protocol LowCardinality(String),
+decoded_path LowCardinality(String),
+dns_message_id Nullable(Int32),
+dns_qr Nullable(Int32),
+dns_opcode Nullable(Int32),
+dns_aa Nullable(Int32),
+dns_tc Nullable(Int32),
+dns_rd Nullable(Int32),
+dns_ra Nullable(Int32),
+dns_rcode Nullable(Int32),
+dns_qdcount Nullable(Int32),
+dns_ancount Nullable(Int32),
+dns_nscount Nullable(Int32),
+dns_arcount Nullable(Int32),
+dns_qname String,
+dns_qtype Nullable(Int32),
+dns_qclass Nullable(Int32),
+dns_cname String,
+dns_sub Nullable(Int32),
+dns_rr String,
+dns_response_latency_ms Nullable(Int32),
+http_url String,
+http_host String,
+http_request_line String,
+http_response_line String,
+http_request_body String,
+http_response_body String,
+http_proxy_flag Nullable(Int32),
+http_sequence Nullable(Int32),
+http_cookie String,
+http_referer String,
+http_user_agent String,
+http_request_content_length Nullable(Int64),
+http_request_content_type String,
+http_response_content_length Nullable(Int64),
+http_response_content_type String,
+http_set_cookie String,
+http_version String,
+http_status_code Nullable(Int32),
+http_response_latency_ms Nullable(Int32),
+http_session_duration_ms Nullable(Int32),
+http_action_file_size Nullable(Int64),
+ssl_version String,
+ssl_sni String,
+ssl_san String,
+ssl_cn String,
+ssl_handshake_latency_ms Nullable(Int32),
+ssl_ja3_hash String,
+ssl_ja3s_hash String,
+ssl_cert_issuer String,
+ssl_cert_subject String,
+ssl_esni_flag Nullable(Int32),
+ssl_ech_flag Nullable(Int32),
+dtls_cookie String,
+dtls_version  String,
+dtls_sni String,
+dtls_san String,
+dtls_cn String,
+dtls_handshake_latency_ms Nullable(Int32),
+dtls_ja3_fingerprint String,
+dtls_ja3_hash String,
+dtls_cert_issuer String,
+dtls_cert_subject String,
+mail_protocol_type String,
+mail_account String,
+mail_from_cmd String,
+mail_to_cmd String,
+mail_from String,
+mail_password String,
+mail_to String,
+mail_cc String,
+mail_bcc String,
+mail_subject String,
+mail_subject_charset String,
+mail_attachment_name String,
+mail_attachment_name_charset String,
+mail_eml_file String,
+ftp_account String,
+ftp_url String,
+ftp_link_type String,
+quic_version String,
+quic_sni String,
+quic_user_agent String,
+rdp_cookie String,
+rdp_security_protocol String,
+rdp_client_channels String,
+rdp_keyboard_layout String,
+rdp_client_version String,
+rdp_client_name String,
+rdp_client_product_id String,
+rdp_desktop_width String,
+rdp_desktop_height String,
+rdp_requested_color_depth String,
+rdp_certificate_type String,
+rdp_certificate_count Nullable(Int32),
+rdp_certificate_permanent Nullable(Int32),
+rdp_encryption_level String,
+rdp_encryption_method String,
+ssh_version String,
+ssh_auth_success String,
+ssh_client_version String,
+ssh_server_version String,
+ssh_cipher_alg String,
+ssh_mac_alg String,
+ssh_compression_alg String,
+ssh_kex_alg String,
+ssh_host_key_alg String,
+ssh_host_key String,
+ssh_hassh String,
+sip_call_id String,
+sip_originator_description String,
+sip_responder_description String,
+sip_user_agent String,
+sip_server String,
+sip_originator_sdp_connect_ip String,
+sip_originator_sdp_media_port Nullable(Int32),
+sip_originator_sdp_media_type String,
+sip_originator_sdp_content String,
+sip_responder_sdp_connect_ip String,
+sip_responder_sdp_media_port Nullable(Int32),
+sip_responder_sdp_media_type String,
+sip_responder_sdp_content String,
+sip_duration_s Nullable(Int32),
+sip_bye String,
+rtp_payload_type_c2s Nullable(Int32),
+rtp_payload_type_s2c Nullable(Int32),
+rtp_pcap_path String,
+rtp_originator_dir Nullable(Int32),
+stratum_cryptocurrency String,
+stratum_mining_pools String,
+stratum_mining_program String,
+stratum_mining_subscribe String,
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+tcp_c2s_ip_fragments Nullable(Int64),
+tcp_s2c_ip_fragments Nullable(Int64),
+tcp_c2s_lost_bytes Nullable(Int64),
+tcp_s2c_lost_bytes Nullable(Int64),
+tcp_c2s_o3_pkts Nullable(Int64),
+tcp_s2c_o3_pkts Nullable(Int64),
+tcp_c2s_rtx_pkts Nullable(Int64),
+tcp_s2c_rtx_pkts Nullable(Int64),
+tcp_c2s_rtx_bytes Nullable(Int64),
+tcp_s2c_rtx_bytes Nullable(Int64),
+tcp_rtt_ms Nullable(Int32),
+tcp_client_isn Nullable(Int64),
+tcp_server_isn Nullable(Int64),
+packet_capture_file String,
+in_src_mac String,
+out_src_mac String,
+in_dest_mac String,
+out_dest_mac String,
+tunnels String,
+dup_traffic_flag Nullable(Int32),
+tunnel_endpoint_a_desc String,
+tunnel_endpoint_b_desc String
+)
+ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,security_event_local,rand());
+
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event_local on cluster ck_cluster (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+start_timestamp_ms DateTime64(3),
+end_timestamp_ms DateTime64(3),
+duration_ms Int32,
+tcp_handshake_latency_ms Nullable(Int32),
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
+device_id String,
+out_link_id Nullable(Int32),
+in_link_id Nullable(Int32),
+device_tag String,
+data_center String,
+device_group String,
+sled_ip String,
+address_type Int32,
+vsys_id Int32,
+t_vsys_id Int32,
+flags Int64,
+flags_identify_info String,
+security_rule_list Array(Int64),
+security_action String,
+monitor_rule_list Array(Int64),
+shaping_rule_list Array(Int64),
+proxy_rule_list Array(Int64),
+statistics_rule_list Array(Int64),
+sc_rule_list Array(Int64),
+sc_rsp_raw Array(Int64),
+sc_rsp_decrypted Array(Int64),
+proxy_action String,
+proxy_pinning_status Nullable(Int32),
+proxy_intercept_status Nullable(Int32),
+proxy_passthrough_reason String,
+proxy_client_side_latency_ms Nullable(Int32),
+proxy_server_side_latency_ms Nullable(Int32),
+proxy_client_side_version String,
+proxy_server_side_version String,
+proxy_cert_verify Nullable(Int32),
+proxy_intercept_error String,
+monitor_mirrored_pkts Nullable(Int32),
+monitor_mirrored_bytes Nullable(Int32),
+client_ip String,
+client_port Int32,
+client_os_desc String,
+client_geolocation LowCardinality(String),
+client_asn Nullable(Int64),
+subscriber_id String,
+imei String,
+imsi String,
+phone_number String,
+apn String,
+server_ip String,
+server_port Int32,
+server_os_desc String,
+server_geolocation LowCardinality(String),
+server_asn Nullable(Int64),
+server_fqdn String,
+server_domain String,
+app_transition String, 
+app LowCardinality(String),
+app_debug_info String,
+app_content String,
+fqdn_category_list Array(Int64),
+ip_protocol LowCardinality(String),
+decoded_path LowCardinality(String),
+dns_message_id Nullable(Int32),
+dns_qr Nullable(Int32),
+dns_opcode Nullable(Int32),
+dns_aa Nullable(Int32),
+dns_tc Nullable(Int32),
+dns_rd Nullable(Int32),
+dns_ra Nullable(Int32),
+dns_rcode Nullable(Int32),
+dns_qdcount Nullable(Int32),
+dns_ancount Nullable(Int32),
+dns_nscount Nullable(Int32),
+dns_arcount Nullable(Int32),
+dns_qname String,
+dns_qtype Nullable(Int32),
+dns_qclass Nullable(Int32),
+dns_cname String,
+dns_sub Nullable(Int32),
+dns_rr String,
+dns_response_latency_ms Nullable(Int32),
+http_url String,
+http_host String,
+http_request_line String,
+http_response_line String,
+http_request_body String,
+http_response_body String,
+http_proxy_flag Nullable(Int32),
+http_sequence Nullable(Int32),
+http_cookie String,
+http_referer String,
+http_user_agent String,
+http_request_content_length Nullable(Int64),
+http_request_content_type String,
+http_response_content_length Nullable(Int64),
+http_response_content_type String,
+http_set_cookie String,
+http_version String,
+http_status_code Nullable(Int32),
+http_response_latency_ms Nullable(Int32),
+http_session_duration_ms Nullable(Int32),
+http_action_file_size Nullable(Int64),
+ssl_version String,
+ssl_sni String,
+ssl_san String,
+ssl_cn String,
+ssl_handshake_latency_ms Nullable(Int32),
+ssl_ja3_hash String,
+ssl_ja3s_hash String,
+ssl_cert_issuer String,
+ssl_cert_subject String,
+ssl_esni_flag Nullable(Int32),
+ssl_ech_flag Nullable(Int32),
+dtls_cookie String,
+dtls_version  String,
+dtls_sni String,
+dtls_san String,
+dtls_cn String,
+dtls_handshake_latency_ms Nullable(Int32),
+dtls_ja3_fingerprint String,
+dtls_ja3_hash String,
+dtls_cert_issuer String,
+dtls_cert_subject String,
+mail_protocol_type String,
+mail_account String,
+mail_from_cmd String,
+mail_to_cmd String,
+mail_from String,
+mail_password String,
+mail_to String,
+mail_cc String,
+mail_bcc String,
+mail_subject String,
+mail_subject_charset String,
+mail_attachment_name String,
+mail_attachment_name_charset String,
+mail_eml_file String,
+ftp_account String,
+ftp_url String,
+ftp_link_type String,
+quic_version String,
+quic_sni String,
+quic_user_agent String,
+rdp_cookie String,
+rdp_security_protocol String,
+rdp_client_channels String,
+rdp_keyboard_layout String,
+rdp_client_version String,
+rdp_client_name String,
+rdp_client_product_id String,
+rdp_desktop_width String,
+rdp_desktop_height String,
+rdp_requested_color_depth String,
+rdp_certificate_type String,
+rdp_certificate_count Nullable(Int32),
+rdp_certificate_permanent Nullable(Int32),
+rdp_encryption_level String,
+rdp_encryption_method String,
+ssh_version String,
+ssh_auth_success String,
+ssh_client_version String,
+ssh_server_version String,
+ssh_cipher_alg String,
+ssh_mac_alg String,
+ssh_compression_alg String,
+ssh_kex_alg String,
+ssh_host_key_alg String,
+ssh_host_key String,
+ssh_hassh String,
+sip_call_id String,
+sip_originator_description String,
+sip_responder_description String,
+sip_user_agent String,
+sip_server String,
+sip_originator_sdp_connect_ip String,
+sip_originator_sdp_media_port Nullable(Int32),
+sip_originator_sdp_media_type String,
+sip_originator_sdp_content String,
+sip_responder_sdp_connect_ip String,
+sip_responder_sdp_media_port Nullable(Int32),
+sip_responder_sdp_media_type String,
+sip_responder_sdp_content String,
+sip_duration_s Nullable(Int32),
+sip_bye String,
+rtp_payload_type_c2s Nullable(Int32),
+rtp_payload_type_s2c Nullable(Int32),
+rtp_pcap_path String,
+rtp_originator_dir Nullable(Int32),
+stratum_cryptocurrency String,
+stratum_mining_pools String,
+stratum_mining_program String,
+stratum_mining_subscribe String,
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+tcp_c2s_ip_fragments Nullable(Int64),
+tcp_s2c_ip_fragments Nullable(Int64),
+tcp_c2s_lost_bytes Nullable(Int64),
+tcp_s2c_lost_bytes Nullable(Int64),
+tcp_c2s_o3_pkts Nullable(Int64),
+tcp_s2c_o3_pkts Nullable(Int64),
+tcp_c2s_rtx_pkts Nullable(Int64),
+tcp_s2c_rtx_pkts Nullable(Int64),
+tcp_c2s_rtx_bytes Nullable(Int64),
+tcp_s2c_rtx_bytes Nullable(Int64),
+tcp_rtt_ms Nullable(Int32),
+tcp_client_isn Nullable(Int64),
+tcp_server_isn Nullable(Int64),
+packet_capture_file String,
+in_src_mac String,
+out_src_mac String,
+in_dest_mac String,
+out_dest_mac String,
+tunnels String,
+dup_traffic_flag Nullable(Int32),
+tunnel_endpoint_a_desc String,
+tunnel_endpoint_b_desc String
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(toDate(recv_time))
+ORDER BY (vsys_id, security_action,proxy_action,decoded_as,data_center, device_group,recv_time);
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event on cluster ck_cluster (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+start_timestamp_ms DateTime64(3),
+end_timestamp_ms DateTime64(3),
+duration_ms Int32,
+tcp_handshake_latency_ms Nullable(Int32),
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64,
+device_id String,
+out_link_id Nullable(Int32),
+in_link_id Nullable(Int32),
+device_tag String,
+data_center String,
+device_group String,
+sled_ip String,
+address_type Int32,
+vsys_id Int32,
+t_vsys_id Int32,
+flags Int64,
+flags_identify_info String,
+security_rule_list Array(Int64),
+security_action String,
+monitor_rule_list Array(Int64),
+shaping_rule_list Array(Int64),
+proxy_rule_list Array(Int64),
+statistics_rule_list Array(Int64),
+sc_rule_list Array(Int64),
+sc_rsp_raw Array(Int64),
+sc_rsp_decrypted Array(Int64),
+proxy_action String,
+proxy_pinning_status Nullable(Int32),
+proxy_intercept_status Nullable(Int32),
+proxy_passthrough_reason String,
+proxy_client_side_latency_ms Nullable(Int32),
+proxy_server_side_latency_ms Nullable(Int32),
+proxy_client_side_version String,
+proxy_server_side_version String,
+proxy_cert_verify Nullable(Int32),
+proxy_intercept_error String,
+monitor_mirrored_pkts Nullable(Int32),
+monitor_mirrored_bytes Nullable(Int32),
+client_ip String,
+client_port Int32,
+client_os_desc String,
+client_geolocation LowCardinality(String),
+client_asn Nullable(Int64),
+subscriber_id String,
+imei String,
+imsi String,
+phone_number String,
+apn String,
+server_ip String,
+server_port Int32,
+server_os_desc String,
+server_geolocation LowCardinality(String),
+server_asn Nullable(Int64),
+server_fqdn String,
+server_domain String,
+app_transition String, 
+app LowCardinality(String),
+app_debug_info String,
+app_content String,
+fqdn_category_list Array(Int64),
+ip_protocol LowCardinality(String),
+decoded_path LowCardinality(String),
+dns_message_id Nullable(Int32),
+dns_qr Nullable(Int32),
+dns_opcode Nullable(Int32),
+dns_aa Nullable(Int32),
+dns_tc Nullable(Int32),
+dns_rd Nullable(Int32),
+dns_ra Nullable(Int32),
+dns_rcode Nullable(Int32),
+dns_qdcount Nullable(Int32),
+dns_ancount Nullable(Int32),
+dns_nscount Nullable(Int32),
+dns_arcount Nullable(Int32),
+dns_qname String,
+dns_qtype Nullable(Int32),
+dns_qclass Nullable(Int32),
+dns_cname String,
+dns_sub Nullable(Int32),
+dns_rr String,
+dns_response_latency_ms Nullable(Int32),
+http_url String,
+http_host String,
+http_request_line String,
+http_response_line String,
+http_request_body String,
+http_response_body String,
+http_proxy_flag Nullable(Int32),
+http_sequence Nullable(Int32),
+http_cookie String,
+http_referer String,
+http_user_agent String,
+http_request_content_length Nullable(Int64),
+http_request_content_type String,
+http_response_content_length Nullable(Int64),
+http_response_content_type String,
+http_set_cookie String,
+http_version String,
+http_status_code Nullable(Int32),
+http_response_latency_ms Nullable(Int32),
+http_session_duration_ms Nullable(Int32),
+http_action_file_size Nullable(Int64),
+ssl_version String,
+ssl_sni String,
+ssl_san String,
+ssl_cn String,
+ssl_handshake_latency_ms Nullable(Int32),
+ssl_ja3_hash String,
+ssl_ja3s_hash String,
+ssl_cert_issuer String,
+ssl_cert_subject String,
+ssl_esni_flag Nullable(Int32),
+ssl_ech_flag Nullable(Int32),
+dtls_cookie String,
+dtls_version  String,
+dtls_sni String,
+dtls_san String,
+dtls_cn String,
+dtls_handshake_latency_ms Nullable(Int32),
+dtls_ja3_fingerprint String,
+dtls_ja3_hash String,
+dtls_cert_issuer String,
+dtls_cert_subject String,
+mail_protocol_type String,
+mail_account String,
+mail_from_cmd String,
+mail_to_cmd String,
+mail_from String,
+mail_password String,
+mail_to String,
+mail_cc String,
+mail_bcc String,
+mail_subject String,
+mail_subject_charset String,
+mail_attachment_name String,
+mail_attachment_name_charset String,
+mail_eml_file String,
+ftp_account String,
+ftp_url String,
+ftp_link_type String,
+quic_version String,
+quic_sni String,
+quic_user_agent String,
+rdp_cookie String,
+rdp_security_protocol String,
+rdp_client_channels String,
+rdp_keyboard_layout String,
+rdp_client_version String,
+rdp_client_name String,
+rdp_client_product_id String,
+rdp_desktop_width String,
+rdp_desktop_height String,
+rdp_requested_color_depth String,
+rdp_certificate_type String,
+rdp_certificate_count Nullable(Int32),
+rdp_certificate_permanent Nullable(Int32),
+rdp_encryption_level String,
+rdp_encryption_method String,
+ssh_version String,
+ssh_auth_success String,
+ssh_client_version String,
+ssh_server_version String,
+ssh_cipher_alg String,
+ssh_mac_alg String,
+ssh_compression_alg String,
+ssh_kex_alg String,
+ssh_host_key_alg String,
+ssh_host_key String,
+ssh_hassh String,
+sip_call_id String,
+sip_originator_description String,
+sip_responder_description String,
+sip_user_agent String,
+sip_server String,
+sip_originator_sdp_connect_ip String,
+sip_originator_sdp_media_port Nullable(Int32),
+sip_originator_sdp_media_type String,
+sip_originator_sdp_content String,
+sip_responder_sdp_connect_ip String,
+sip_responder_sdp_media_port Nullable(Int32),
+sip_responder_sdp_media_type String,
+sip_responder_sdp_content String,
+sip_duration_s Nullable(Int32),
+sip_bye String,
+rtp_payload_type_c2s Nullable(Int32),
+rtp_payload_type_s2c Nullable(Int32),
+rtp_pcap_path String,
+rtp_originator_dir Nullable(Int32),
+stratum_cryptocurrency String,
+stratum_mining_pools String,
+stratum_mining_program String,
+stratum_mining_subscribe String,
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+tcp_c2s_ip_fragments Nullable(Int64),
+tcp_s2c_ip_fragments Nullable(Int64),
+tcp_c2s_lost_bytes Nullable(Int64),
+tcp_s2c_lost_bytes Nullable(Int64),
+tcp_c2s_o3_pkts Nullable(Int64),
+tcp_s2c_o3_pkts Nullable(Int64),
+tcp_c2s_rtx_pkts Nullable(Int64),
+tcp_s2c_rtx_pkts Nullable(Int64),
+tcp_c2s_rtx_bytes Nullable(Int64),
+tcp_s2c_rtx_bytes Nullable(Int64),
+tcp_rtt_ms Nullable(Int32),
+tcp_client_isn Nullable(Int64),
+tcp_server_isn Nullable(Int64),
+packet_capture_file String,
+in_src_mac String,
+out_src_mac String,
+in_dest_mac String,
+out_dest_mac String,
+tunnels String,
+dup_traffic_flag Nullable(Int32),
+tunnel_endpoint_a_desc String,
+tunnel_endpoint_b_desc String
+)
+ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,monitor_event_local,rand());
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.monitor_event on cluster ck_query (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+start_timestamp_ms DateTime64(3),
+end_timestamp_ms DateTime64(3),
+duration_ms Int32,
+tcp_handshake_latency_ms Nullable(Int32),
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64 ,
+device_id String,
+out_link_id Nullable(Int32),
+in_link_id Nullable(Int32),
+device_tag String,
+data_center String,
+device_group String,
+sled_ip String,
+address_type Int32,
+vsys_id Int32,
+t_vsys_id Int32,
+flags Int64,
+flags_identify_info String,
+security_rule_list Array(Int64),
+security_action String,
+monitor_rule_list Array(Int64),
+shaping_rule_list Array(Int64),
+proxy_rule_list Array(Int64),
+statistics_rule_list Array(Int64),
+sc_rule_list Array(Int64),
+sc_rsp_raw Array(Int64),
+sc_rsp_decrypted Array(Int64),
+proxy_action String,
+proxy_pinning_status Nullable(Int32),
+proxy_intercept_status Nullable(Int32),
+proxy_passthrough_reason String,
+proxy_client_side_latency_ms Nullable(Int32),
+proxy_server_side_latency_ms Nullable(Int32),
+proxy_client_side_version String,
+proxy_server_side_version String,
+proxy_cert_verify Nullable(Int32),
+proxy_intercept_error String,
+monitor_mirrored_pkts Nullable(Int32),
+monitor_mirrored_bytes Nullable(Int32),
+client_ip String,
+client_port Int32,
+client_os_desc String,
+client_geolocation LowCardinality(String),
+client_asn Nullable(Int64),
+subscriber_id String,
+imei String,
+imsi String,
+phone_number String,
+apn String,
+server_ip String,
+server_port Int32,
+server_os_desc String,
+server_geolocation LowCardinality(String),
+server_asn Nullable(Int64),
+server_fqdn String,
+server_domain String,
+app_transition String, 
+app LowCardinality(String),
+app_debug_info String,
+app_content String,
+fqdn_category_list Array(Int64),
+ip_protocol LowCardinality(String),
+decoded_path LowCardinality(String),
+dns_message_id Nullable(Int32),
+dns_qr Nullable(Int32),
+dns_opcode Nullable(Int32),
+dns_aa Nullable(Int32),
+dns_tc Nullable(Int32),
+dns_rd Nullable(Int32),
+dns_ra Nullable(Int32),
+dns_rcode Nullable(Int32),
+dns_qdcount Nullable(Int32),
+dns_ancount Nullable(Int32),
+dns_nscount Nullable(Int32),
+dns_arcount Nullable(Int32),
+dns_qname String,
+dns_qtype Nullable(Int32),
+dns_qclass Nullable(Int32),
+dns_cname String,
+dns_sub Nullable(Int32),
+dns_rr String,
+dns_response_latency_ms Nullable(Int32),
+http_url String,
+http_host String,
+http_request_line String,
+http_response_line String,
+http_request_body String,
+http_response_body String,
+http_proxy_flag Nullable(Int32),
+http_sequence Nullable(Int32),
+http_cookie String,
+http_referer String,
+http_user_agent String,
+http_request_content_length Nullable(Int64),
+http_request_content_type String,
+http_response_content_length Nullable(Int64),
+http_response_content_type String,
+http_set_cookie String,
+http_version String,
+http_status_code Nullable(Int32),
+http_response_latency_ms Nullable(Int32),
+http_session_duration_ms Nullable(Int32),
+http_action_file_size Nullable(Int64),
+ssl_version String,
+ssl_sni String,
+ssl_san String,
+ssl_cn String,
+ssl_handshake_latency_ms Nullable(Int32),
+ssl_ja3_hash String,
+ssl_ja3s_hash String,
+ssl_cert_issuer String,
+ssl_cert_subject String,
+ssl_esni_flag Nullable(Int32),
+ssl_ech_flag Nullable(Int32),
+dtls_cookie String,
+dtls_version  String,
+dtls_sni String,
+dtls_san String,
+dtls_cn String,
+dtls_handshake_latency_ms Nullable(Int32),
+dtls_ja3_fingerprint String,
+dtls_ja3_hash String,
+dtls_cert_issuer String,
+dtls_cert_subject String,
+mail_protocol_type String,
+mail_account String,
+mail_from_cmd String,
+mail_to_cmd String,
+mail_from String,
+mail_password String,
+mail_to String,
+mail_cc String,
+mail_bcc String,
+mail_subject String,
+mail_subject_charset String,
+mail_attachment_name String,
+mail_attachment_name_charset String,
+mail_eml_file String,
+ftp_account String,
+ftp_url String,
+ftp_link_type String,
+quic_version String,
+quic_sni String,
+quic_user_agent String,
+rdp_cookie String,
+rdp_security_protocol String,
+rdp_client_channels String,
+rdp_keyboard_layout String,
+rdp_client_version String,
+rdp_client_name String,
+rdp_client_product_id String,
+rdp_desktop_width String,
+rdp_desktop_height String,
+rdp_requested_color_depth String,
+rdp_certificate_type String,
+rdp_certificate_count Nullable(Int32),
+rdp_certificate_permanent Nullable(Int32),
+rdp_encryption_level String,
+rdp_encryption_method String,
+ssh_version String,
+ssh_auth_success String,
+ssh_client_version String,
+ssh_server_version String,
+ssh_cipher_alg String,
+ssh_mac_alg String,
+ssh_compression_alg String,
+ssh_kex_alg String,
+ssh_host_key_alg String,
+ssh_host_key String,
+ssh_hassh String,
+sip_call_id String,
+sip_originator_description String,
+sip_responder_description String,
+sip_user_agent String,
+sip_server String,
+sip_originator_sdp_connect_ip String,
+sip_originator_sdp_media_port Nullable(Int32),
+sip_originator_sdp_media_type String,
+sip_originator_sdp_content String,
+sip_responder_sdp_connect_ip String,
+sip_responder_sdp_media_port Nullable(Int32),
+sip_responder_sdp_media_type String,
+sip_responder_sdp_content String,
+sip_duration_s Nullable(Int32),
+sip_bye String,
+rtp_payload_type_c2s Nullable(Int32),
+rtp_payload_type_s2c Nullable(Int32),
+rtp_pcap_path String,
+rtp_originator_dir Nullable(Int32),
+stratum_cryptocurrency String,
+stratum_mining_pools String,
+stratum_mining_program String,
+stratum_mining_subscribe String,
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+tcp_c2s_ip_fragments Nullable(Int64),
+tcp_s2c_ip_fragments Nullable(Int64),
+tcp_c2s_lost_bytes Nullable(Int64),
+tcp_s2c_lost_bytes Nullable(Int64),
+tcp_c2s_o3_pkts Nullable(Int64),
+tcp_s2c_o3_pkts Nullable(Int64),
+tcp_c2s_rtx_pkts Nullable(Int64),
+tcp_s2c_rtx_pkts Nullable(Int64),
+tcp_c2s_rtx_bytes Nullable(Int64),
+tcp_s2c_rtx_bytes Nullable(Int64),
+tcp_rtt_ms Nullable(Int32),
+tcp_client_isn Nullable(Int64),
+tcp_server_isn Nullable(Int64),
+packet_capture_file String,
+in_src_mac String,
+out_src_mac String,
+in_dest_mac String,
+out_dest_mac String,
+tunnels String,
+dup_traffic_flag Nullable(Int32),
+tunnel_endpoint_a_desc String,
+tunnel_endpoint_b_desc String
+)
+ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,monitor_event_local,rand());
+
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record_local on cluster ck_cluster (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
+address_type Int32,
+vsys_id Int32,
+client_ip String,
+client_port Int32,
+server_ip String,
+server_port Int32,
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+dns_message_id Nullable(Int32),
+dns_qr Nullable(Int32),
+dns_opcode Nullable(Int32),
+dns_aa Nullable(Int32),
+dns_tc Nullable(Int32),
+dns_rd Nullable(Int32),
+dns_ra Nullable(Int32),
+dns_rcode Nullable(Int32),
+dns_qdcount Nullable(Int32),
+dns_ancount Nullable(Int32),
+dns_nscount Nullable(Int32),
+dns_arcount Nullable(Int32),
+dns_qname String,
+dns_qtype Nullable(Int32),
+dns_qclass Nullable(Int32),
+dns_cname String,
+dns_sub Nullable(Int32),
+dns_rr String,
+dns_response_latency_ms Nullable(Int32),
+http_url String,
+http_host String,
+http_request_line String,
+http_response_line String,
+http_request_body String,
+http_response_body String,
+http_proxy_flag Nullable(Int32),
+http_sequence Nullable(Int32),
+http_cookie String,
+http_referer String,
+http_user_agent String,
+http_request_content_length Nullable(Int64),
+http_request_content_type String,
+http_response_content_length Nullable(Int64),
+http_response_content_type String,
+http_set_cookie String,
+http_version String,
+http_status_code Nullable(Int32),
+http_response_latency_ms Nullable(Int32),
+http_session_duration_ms Nullable(Int32),
+http_action_file_size Nullable(Int64),
+mail_protocol_type String,
+mail_account String,
+mail_from_cmd String,
+mail_to_cmd String,
+mail_from String,
+mail_password String,
+mail_to String,
+mail_cc String,
+mail_bcc String,
+mail_subject String,
+mail_subject_charset String,
+mail_attachment_name String,
+mail_attachment_name_charset String,
+mail_eml_file String,
+sip_call_id String,
+sip_originator_description String,
+sip_responder_description String,
+sip_user_agent String,
+sip_server String,
+sip_originator_sdp_connect_ip String,
+sip_originator_sdp_media_port Nullable(Int32),
+sip_originator_sdp_media_type String,
+sip_originator_sdp_content String,
+sip_responder_sdp_connect_ip String,
+sip_responder_sdp_media_port Nullable(Int32),
+sip_responder_sdp_media_type String,
+sip_responder_sdp_content String,
+sip_duration_s Nullable(Int32),
+sip_bye String
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(toDate(recv_time))
+ORDER BY (vsys_id,session_id,recv_time);
+
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_cluster (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64 ,
+address_type Int32,
+vsys_id Int32,
+client_ip String,
+client_port Int32,
+server_ip String,
+server_port Int32,
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+dns_message_id Nullable(Int32),
+dns_qr Nullable(Int32),
+dns_opcode Nullable(Int32),
+dns_aa Nullable(Int32),
+dns_tc Nullable(Int32),
+dns_rd Nullable(Int32),
+dns_ra Nullable(Int32),
+dns_rcode Nullable(Int32),
+dns_qdcount Nullable(Int32),
+dns_ancount Nullable(Int32),
+dns_nscount Nullable(Int32),
+dns_arcount Nullable(Int32),
+dns_qname String,
+dns_qtype Nullable(Int32),
+dns_qclass Nullable(Int32),
+dns_cname String,
+dns_sub Nullable(Int32),
+dns_rr String,
+dns_response_latency_ms Nullable(Int32),
+http_url String,
+http_host String,
+http_request_line String,
+http_response_line String,
+http_request_body String,
+http_response_body String,
+http_proxy_flag Nullable(Int32),
+http_sequence Nullable(Int32),
+http_cookie String,
+http_referer String,
+http_user_agent String,
+http_request_content_length Nullable(Int64),
+http_request_content_type String,
+http_response_content_length Nullable(Int64),
+http_response_content_type String,
+http_set_cookie String,
+http_version String,
+http_status_code Nullable(Int32),
+http_response_latency_ms Nullable(Int32),
+http_session_duration_ms Nullable(Int32),
+http_action_file_size Nullable(Int64),
+mail_protocol_type String,
+mail_account String,
+mail_from_cmd String,
+mail_to_cmd String,
+mail_from String,
+mail_password String,
+mail_to String,
+mail_cc String,
+mail_bcc String,
+mail_subject String,
+mail_subject_charset String,
+mail_attachment_name String,
+mail_attachment_name_charset String,
+mail_eml_file String,
+sip_call_id String,
+sip_originator_description String,
+sip_responder_description String,
+sip_user_agent String,
+sip_server String,
+sip_originator_sdp_connect_ip String,
+sip_originator_sdp_media_port Nullable(Int32),
+sip_originator_sdp_media_type String,
+sip_originator_sdp_content String,
+sip_responder_sdp_connect_ip String,
+sip_responder_sdp_media_port Nullable(Int32),
+sip_responder_sdp_media_type String,
+sip_responder_sdp_content String,
+sip_duration_s Nullable(Int32),
+sip_bye String
+)
+ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand());
+
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.transaction_record on cluster ck_query (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64 ,
+address_type Int32,
+vsys_id Int32,
+client_ip String,
+client_port Int32,
+server_ip String,
+server_port Int32,
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+dns_message_id Nullable(Int32),
+dns_qr Nullable(Int32),
+dns_opcode Nullable(Int32),
+dns_aa Nullable(Int32),
+dns_tc Nullable(Int32),
+dns_rd Nullable(Int32),
+dns_ra Nullable(Int32),
+dns_rcode Nullable(Int32),
+dns_qdcount Nullable(Int32),
+dns_ancount Nullable(Int32),
+dns_nscount Nullable(Int32),
+dns_arcount Nullable(Int32),
+dns_qname String,
+dns_qtype Nullable(Int32),
+dns_qclass Nullable(Int32),
+dns_cname String,
+dns_sub Nullable(Int32),
+dns_rr String,
+dns_response_latency_ms Nullable(Int32),
+http_url String,
+http_host String,
+http_request_line String,
+http_response_line String,
+http_request_body String,
+http_response_body String,
+http_proxy_flag Nullable(Int32),
+http_sequence Nullable(Int32),
+http_cookie String,
+http_referer String,
+http_user_agent String,
+http_request_content_length Nullable(Int64),
+http_request_content_type String,
+http_response_content_length Nullable(Int64),
+http_response_content_type String,
+http_set_cookie String,
+http_version String,
+http_status_code Nullable(Int32),
+http_response_latency_ms Nullable(Int32),
+http_session_duration_ms Nullable(Int32),
+http_action_file_size Nullable(Int64),
+mail_protocol_type String,
+mail_account String,
+mail_from_cmd String,
+mail_to_cmd String,
+mail_from String,
+mail_password String,
+mail_to String,
+mail_cc String,
+mail_bcc String,
+mail_subject String,
+mail_subject_charset String,
+mail_attachment_name String,
+mail_attachment_name_charset String,
+mail_eml_file String,
+sip_call_id String,
+sip_originator_description String,
+sip_responder_description String,
+sip_user_agent String,
+sip_server String,
+sip_originator_sdp_connect_ip String,
+sip_originator_sdp_media_port Nullable(Int32),
+sip_originator_sdp_media_type String,
+sip_originator_sdp_content String,
+sip_responder_sdp_connect_ip String,
+sip_responder_sdp_media_port Nullable(Int32),
+sip_responder_sdp_media_type String,
+sip_responder_sdp_content String,
+sip_duration_s Nullable(Int32),
+sip_bye String
+)
+ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,transaction_record_local,rand());
+
+
+alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1;
+alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add INDEX IF NOT EXISTS client_index client_ip type bloom_filter(0.05) GRANULARITY 1;
+
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record_local on cluster ck_cluster (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+start_timestamp_ms DateTime64(3),
+end_timestamp_ms DateTime64(3),
+duration_ms Int32,
+tcp_handshake_latency_ms Nullable(Int32),
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
+device_id String,
+out_link_id Nullable(Int32),
+in_link_id Nullable(Int32),
+device_tag String,
+data_center String,
+device_group String,
+sled_ip String,
+address_type Int32,
+vsys_id Int32,
+t_vsys_id Int32,
+flags Int64,
+flags_identify_info String,
+security_rule_list Array(Int64),
+security_action String,
+monitor_rule_list Array(Int64),
+shaping_rule_list Array(Int64),
+proxy_rule_list Array(Int64),
+statistics_rule_list Array(Int64),
+sc_rule_list Array(Int64),
+sc_rsp_raw Array(Int64),
+sc_rsp_decrypted Array(Int64),
+proxy_action String,
+proxy_pinning_status Nullable(Int32),
+proxy_intercept_status Nullable(Int32),
+proxy_passthrough_reason String,
+proxy_client_side_latency_ms Nullable(Int32),
+proxy_server_side_latency_ms Nullable(Int32),
+proxy_client_side_version String,
+proxy_server_side_version String,
+proxy_cert_verify Nullable(Int32),
+proxy_intercept_error String,
+monitor_mirrored_pkts Nullable(Int32),
+monitor_mirrored_bytes Nullable(Int32),
+client_ip String,
+client_port Int32,
+client_os_desc String,
+client_geolocation LowCardinality(String),
+client_asn Nullable(Int64),
+subscriber_id String,
+imei String,
+imsi String,
+phone_number String,
+apn String,
+server_ip String,
+server_port Int32,
+server_os_desc String,
+server_geolocation LowCardinality(String),
+server_asn Nullable(Int64),
+server_fqdn String,
+server_domain String,
+app_transition String, 
+app LowCardinality(String),
+app_debug_info String,
+app_content String,
+fqdn_category_list Array(Int64),
+ip_protocol LowCardinality(String),
+decoded_path LowCardinality(String),
+sip_call_id String,
+sip_originator_description String,
+sip_responder_description String,
+sip_user_agent String,
+sip_server String,
+sip_originator_sdp_connect_ip String,
+sip_originator_sdp_media_port Nullable(Int32),
+sip_originator_sdp_media_type String,
+sip_originator_sdp_content String,
+sip_responder_sdp_connect_ip String,
+sip_responder_sdp_media_port Nullable(Int32),
+sip_responder_sdp_media_type String,
+sip_responder_sdp_content String,
+sip_duration_s Nullable(Int32),
+sip_bye String,
+rtp_payload_type_c2s Nullable(Int32),
+rtp_payload_type_s2c Nullable(Int32),
+rtp_pcap_path String,
+rtp_originator_dir Nullable(Int32),
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+tcp_c2s_ip_fragments Nullable(Int64),
+tcp_s2c_ip_fragments Nullable(Int64),
+tcp_c2s_lost_bytes Nullable(Int64),
+tcp_s2c_lost_bytes Nullable(Int64),
+tcp_c2s_o3_pkts Nullable(Int64),
+tcp_s2c_o3_pkts Nullable(Int64),
+tcp_c2s_rtx_pkts Nullable(Int64),
+tcp_s2c_rtx_pkts Nullable(Int64),
+tcp_c2s_rtx_bytes Nullable(Int64),
+tcp_s2c_rtx_bytes Nullable(Int64),
+tcp_rtt_ms Nullable(Int32),
+tcp_client_isn Nullable(Int64),
+tcp_server_isn Nullable(Int64),
+packet_capture_file String,
+in_src_mac String,
+out_src_mac String,
+in_dest_mac String,
+out_dest_mac String,
+tunnels String,
+dup_traffic_flag Nullable(Int32),
+tunnel_endpoint_a_desc String,
+tunnel_endpoint_b_desc String
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(toDate(recv_time))
+ORDER BY (vsys_id,decoded_as,data_center, device_group,recv_time);
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record on cluster ck_cluster (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+start_timestamp_ms DateTime64(3),
+end_timestamp_ms DateTime64(3),
+duration_ms Int32,
+tcp_handshake_latency_ms Nullable(Int32),
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64,
+device_id String,
+out_link_id Nullable(Int32),
+in_link_id Nullable(Int32),
+device_tag String,
+data_center String,
+device_group String,
+sled_ip String,
+address_type Int32,
+vsys_id Int32,
+t_vsys_id Int32,
+flags Int64,
+flags_identify_info String,
+security_rule_list Array(Int64),
+security_action String,
+monitor_rule_list Array(Int64),
+shaping_rule_list Array(Int64),
+proxy_rule_list Array(Int64),
+statistics_rule_list Array(Int64),
+sc_rule_list Array(Int64),
+sc_rsp_raw Array(Int64),
+sc_rsp_decrypted Array(Int64),
+proxy_action String,
+proxy_pinning_status Nullable(Int32),
+proxy_intercept_status Nullable(Int32),
+proxy_passthrough_reason String,
+proxy_client_side_latency_ms Nullable(Int32),
+proxy_server_side_latency_ms Nullable(Int32),
+proxy_client_side_version String,
+proxy_server_side_version String,
+proxy_cert_verify Nullable(Int32),
+proxy_intercept_error String,
+monitor_mirrored_pkts Nullable(Int32),
+monitor_mirrored_bytes Nullable(Int32),
+client_ip String,
+client_port Int32,
+client_os_desc String,
+client_geolocation LowCardinality(String),
+client_asn Nullable(Int64),
+subscriber_id String,
+imei String,
+imsi String,
+phone_number String,
+apn String,
+server_ip String,
+server_port Int32,
+server_os_desc String,
+server_geolocation LowCardinality(String),
+server_asn Nullable(Int64),
+server_fqdn String,
+server_domain String,
+app_transition String, 
+app LowCardinality(String),
+app_debug_info String,
+app_content String,
+fqdn_category_list Array(Int64),
+ip_protocol LowCardinality(String),
+decoded_path LowCardinality(String),
+sip_call_id String,
+sip_originator_description String,
+sip_responder_description String,
+sip_user_agent String,
+sip_server String,
+sip_originator_sdp_connect_ip String,
+sip_originator_sdp_media_port Nullable(Int32),
+sip_originator_sdp_media_type String,
+sip_originator_sdp_content String,
+sip_responder_sdp_connect_ip String,
+sip_responder_sdp_media_port Nullable(Int32),
+sip_responder_sdp_media_type String,
+sip_responder_sdp_content String,
+sip_duration_s Nullable(Int32),
+sip_bye String,
+rtp_payload_type_c2s Nullable(Int32),
+rtp_payload_type_s2c Nullable(Int32),
+rtp_pcap_path String,
+rtp_originator_dir Nullable(Int32),
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+tcp_c2s_ip_fragments Nullable(Int64),
+tcp_s2c_ip_fragments Nullable(Int64),
+tcp_c2s_lost_bytes Nullable(Int64),
+tcp_s2c_lost_bytes Nullable(Int64),
+tcp_c2s_o3_pkts Nullable(Int64),
+tcp_s2c_o3_pkts Nullable(Int64),
+tcp_c2s_rtx_pkts Nullable(Int64),
+tcp_s2c_rtx_pkts Nullable(Int64),
+tcp_c2s_rtx_bytes Nullable(Int64),
+tcp_s2c_rtx_bytes Nullable(Int64),
+tcp_rtt_ms Nullable(Int32),
+tcp_client_isn Nullable(Int64),
+tcp_server_isn Nullable(Int64),
+packet_capture_file String,
+in_src_mac String,
+out_src_mac String,
+in_dest_mac String,
+out_dest_mac String,
+tunnels String,
+dup_traffic_flag Nullable(Int32),
+tunnel_endpoint_a_desc String,
+tunnel_endpoint_b_desc String
+)
+ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand());
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.voip_record on cluster ck_query (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+start_timestamp_ms DateTime64(3),
+end_timestamp_ms DateTime64(3),
+duration_ms Int32,
+tcp_handshake_latency_ms Nullable(Int32),
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64,
+device_id String,
+out_link_id Nullable(Int32),
+in_link_id Nullable(Int32),
+device_tag String,
+data_center String,
+device_group String,
+sled_ip String,
+address_type Int32,
+vsys_id Int32,
+t_vsys_id Int32,
+flags Int64,
+flags_identify_info String,
+security_rule_list Array(Int64),
+security_action String,
+monitor_rule_list Array(Int64),
+shaping_rule_list Array(Int64),
+proxy_rule_list Array(Int64),
+statistics_rule_list Array(Int64),
+sc_rule_list Array(Int64),
+sc_rsp_raw Array(Int64),
+sc_rsp_decrypted Array(Int64),
+proxy_action String,
+proxy_pinning_status Nullable(Int32),
+proxy_intercept_status Nullable(Int32),
+proxy_passthrough_reason String,
+proxy_client_side_latency_ms Nullable(Int32),
+proxy_server_side_latency_ms Nullable(Int32),
+proxy_client_side_version String,
+proxy_server_side_version String,
+proxy_cert_verify Nullable(Int32),
+proxy_intercept_error String,
+monitor_mirrored_pkts Nullable(Int32),
+monitor_mirrored_bytes Nullable(Int32),
+client_ip String,
+client_port Int32,
+client_os_desc String,
+client_geolocation LowCardinality(String),
+client_asn Nullable(Int64),
+subscriber_id String,
+imei String,
+imsi String,
+phone_number String,
+apn String,
+server_ip String,
+server_port Int32,
+server_os_desc String,
+server_geolocation LowCardinality(String),
+server_asn Nullable(Int64),
+server_fqdn String,
+server_domain String,
+app_transition String, 
+app LowCardinality(String),
+app_debug_info String,
+app_content String,
+fqdn_category_list Array(Int64),
+ip_protocol LowCardinality(String),
+decoded_path LowCardinality(String),
+sip_call_id String,
+sip_originator_description String,
+sip_responder_description String,
+sip_user_agent String,
+sip_server String,
+sip_originator_sdp_connect_ip String,
+sip_originator_sdp_media_port Nullable(Int32),
+sip_originator_sdp_media_type String,
+sip_originator_sdp_content String,
+sip_responder_sdp_connect_ip String,
+sip_responder_sdp_media_port Nullable(Int32),
+sip_responder_sdp_media_type String,
+sip_responder_sdp_content String,
+sip_duration_s Nullable(Int32),
+sip_bye String,
+rtp_payload_type_c2s Nullable(Int32),
+rtp_payload_type_s2c Nullable(Int32),
+rtp_pcap_path String,
+rtp_originator_dir Nullable(Int32),
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+tcp_c2s_ip_fragments Nullable(Int64),
+tcp_s2c_ip_fragments Nullable(Int64),
+tcp_c2s_lost_bytes Nullable(Int64),
+tcp_s2c_lost_bytes Nullable(Int64),
+tcp_c2s_o3_pkts Nullable(Int64),
+tcp_s2c_o3_pkts Nullable(Int64),
+tcp_c2s_rtx_pkts Nullable(Int64),
+tcp_s2c_rtx_pkts Nullable(Int64),
+tcp_c2s_rtx_bytes Nullable(Int64),
+tcp_s2c_rtx_bytes Nullable(Int64),
+tcp_rtt_ms Nullable(Int32),
+tcp_client_isn Nullable(Int64),
+tcp_server_isn Nullable(Int64),
+packet_capture_file String,
+in_src_mac String,
+out_src_mac String,
+in_dest_mac String,
+out_dest_mac String,
+tunnels String,
+dup_traffic_flag Nullable(Int32),
+tunnel_endpoint_a_desc String,
+tunnel_endpoint_b_desc String
+)
+ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,voip_record_local,rand());
+
+
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event_local on cluster ck_cluster (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+start_timestamp_ms DateTime64(3),
+end_timestamp_ms DateTime64(3),
+duration_ms Int32,
+tcp_handshake_latency_ms Nullable(Int32),
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
+device_id String,
+out_link_id Nullable(Int32),
+in_link_id Nullable(Int32),
+device_tag String,
+data_center String,
+device_group String,
+sled_ip String,
+address_type Int32,
+vsys_id Int32,
+t_vsys_id Int32,
+flags Int64,
+flags_identify_info String,
+security_rule_list Array(Int64),
+security_action String,
+monitor_rule_list Array(Int64),
+shaping_rule_list Array(Int64),
+proxy_rule_list Array(Int64),
+statistics_rule_list Array(Int64),
+sc_rule_list Array(Int64),
+sc_rsp_raw Array(Int64),
+sc_rsp_decrypted Array(Int64),
+proxy_action String,
+proxy_pinning_status Nullable(Int32),
+proxy_intercept_status Nullable(Int32),
+proxy_passthrough_reason String,
+proxy_client_side_latency_ms Nullable(Int32),
+proxy_server_side_latency_ms Nullable(Int32),
+proxy_client_side_version String,
+proxy_server_side_version String,
+proxy_cert_verify Nullable(Int32),
+proxy_intercept_error String,
+monitor_mirrored_pkts Nullable(Int32),
+monitor_mirrored_bytes Nullable(Int32),
+client_ip String,
+client_port Int32,
+client_os_desc String,
+client_geolocation LowCardinality(String),
+client_asn Nullable(Int64),
+subscriber_id String,
+imei String,
+imsi String,
+phone_number String,
+apn String,
+server_ip String,
+server_port Int32,
+server_os_desc String,
+server_geolocation LowCardinality(String),
+server_asn Nullable(Int64),
+server_fqdn String,
+server_domain String,
+app_transition String, 
+app LowCardinality(String),
+app_debug_info String,
+app_content String,
+fqdn_category_list Array(Int64),
+ip_protocol LowCardinality(String),
+decoded_path LowCardinality(String),
+http_url String,
+http_host String,
+http_request_line String,
+http_response_line String,
+http_request_body String,
+http_response_body String,
+http_proxy_flag Nullable(Int32),
+http_sequence Nullable(Int32),
+http_cookie String,
+http_referer String,
+http_user_agent String,
+http_request_content_length Nullable(Int64),
+http_request_content_type String,
+http_response_content_length Nullable(Int64),
+http_response_content_type String,
+http_set_cookie String,
+http_version String,
+http_status_code Nullable(Int32),
+http_response_latency_ms Nullable(Int32),
+http_session_duration_ms Nullable(Int32),
+http_action_file_size Nullable(Int64),
+doh_url String,
+doh_host String,
+doh_request_line String,
+doh_response_line String,
+doh_cookie String,
+doh_referer String,
+doh_user_agent String,
+doh_content_length String,
+doh_content_type String,
+doh_set_cookie String,
+doh_version String,
+doh_message_id Int64,
+doh_qr Nullable(Int64),
+doh_opcode Nullable(Int64),
+doh_aa Nullable(Int64),
+doh_tc Nullable(Int64),
+doh_rd Nullable(Int64),
+doh_ra Nullable(Int64),
+doh_rcode Nullable(Int64),
+doh_qdcount Nullable(Int64),
+doh_ancount Nullable(Int64),
+doh_nscount Nullable(Int64),
+doh_arcount Nullable(Int64),
+doh_qname String,
+doh_qtype Nullable(Int64),
+doh_qclass Nullable(Int64),
+doh_cname String,
+doh_sub Nullable(Int64),
+doh_rr String,
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+tcp_c2s_ip_fragments Nullable(Int64),
+tcp_s2c_ip_fragments Nullable(Int64),
+tcp_c2s_lost_bytes Nullable(Int64),
+tcp_s2c_lost_bytes Nullable(Int64),
+tcp_c2s_o3_pkts Nullable(Int64),
+tcp_s2c_o3_pkts Nullable(Int64),
+tcp_c2s_rtx_pkts Nullable(Int64),
+tcp_s2c_rtx_pkts Nullable(Int64),
+tcp_c2s_rtx_bytes Nullable(Int64),
+tcp_s2c_rtx_bytes Nullable(Int64),
+tcp_rtt_ms Nullable(Int32),
+tcp_client_isn Nullable(Int64),
+tcp_server_isn Nullable(Int64),
+packet_capture_file String,
+in_src_mac String,
+out_src_mac String,
+in_dest_mac String,
+out_dest_mac String,
+tunnels String,
+dup_traffic_flag Nullable(Int32),
+tunnel_endpoint_a_desc String,
+tunnel_endpoint_b_desc String
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(toDate(recv_time))
+ORDER BY (vsys_id,proxy_action,decoded_as,data_center, device_group,recv_time);
+
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event on cluster ck_cluster (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+start_timestamp_ms DateTime64(3),
+end_timestamp_ms DateTime64(3),
+duration_ms Int32,
+tcp_handshake_latency_ms Nullable(Int32),
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64,
+device_id String,
+out_link_id Nullable(Int32),
+in_link_id Nullable(Int32),
+device_tag String,
+data_center String,
+device_group String,
+sled_ip String,
+address_type Int32,
+vsys_id Int32,
+t_vsys_id Int32,
+flags Int64,
+flags_identify_info String,
+security_rule_list Array(Int64),
+security_action String,
+monitor_rule_list Array(Int64),
+shaping_rule_list Array(Int64),
+proxy_rule_list Array(Int64),
+statistics_rule_list Array(Int64),
+sc_rule_list Array(Int64),
+sc_rsp_raw Array(Int64),
+sc_rsp_decrypted Array(Int64),
+proxy_action String,
+proxy_pinning_status Nullable(Int32),
+proxy_intercept_status Nullable(Int32),
+proxy_passthrough_reason String,
+proxy_client_side_latency_ms Nullable(Int32),
+proxy_server_side_latency_ms Nullable(Int32),
+proxy_client_side_version String,
+proxy_server_side_version String,
+proxy_cert_verify Nullable(Int32),
+proxy_intercept_error String,
+monitor_mirrored_pkts Nullable(Int32),
+monitor_mirrored_bytes Nullable(Int32),
+client_ip String,
+client_port Int32,
+client_os_desc String,
+client_geolocation LowCardinality(String),
+client_asn Nullable(Int64),
+subscriber_id String,
+imei String,
+imsi String,
+phone_number String,
+apn String,
+server_ip String,
+server_port Int32,
+server_os_desc String,
+server_geolocation LowCardinality(String),
+server_asn Nullable(Int64),
+server_fqdn String,
+server_domain String,
+app_transition String, 
+app LowCardinality(String),
+app_debug_info String,
+app_content String,
+fqdn_category_list Array(Int64),
+ip_protocol LowCardinality(String),
+decoded_path LowCardinality(String),
+http_url String,
+http_host String,
+http_request_line String,
+http_response_line String,
+http_request_body String,
+http_response_body String,
+http_proxy_flag Nullable(Int32),
+http_sequence Nullable(Int32),
+http_cookie String,
+http_referer String,
+http_user_agent String,
+http_request_content_length Nullable(Int64),
+http_request_content_type String,
+http_response_content_length Nullable(Int64),
+http_response_content_type String,
+http_set_cookie String,
+http_version String,
+http_status_code Nullable(Int32),
+http_response_latency_ms Nullable(Int32),
+http_session_duration_ms Nullable(Int32),
+http_action_file_size Nullable(Int64),
+doh_url String,
+doh_host String,
+doh_request_line String,
+doh_response_line String,
+doh_cookie String,
+doh_referer String,
+doh_user_agent String,
+doh_content_length String,
+doh_content_type String,
+doh_set_cookie String,
+doh_version String,
+doh_message_id Int64,
+doh_qr Nullable(Int64),
+doh_opcode Nullable(Int64),
+doh_aa Nullable(Int64),
+doh_tc Nullable(Int64),
+doh_rd Nullable(Int64),
+doh_ra Nullable(Int64),
+doh_rcode Nullable(Int64),
+doh_qdcount Nullable(Int64),
+doh_ancount Nullable(Int64),
+doh_nscount Nullable(Int64),
+doh_arcount Nullable(Int64),
+doh_qname String,
+doh_qtype Nullable(Int64),
+doh_qclass Nullable(Int64),
+doh_cname String,
+doh_sub Nullable(Int64),
+doh_rr String,
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+tcp_c2s_ip_fragments Nullable(Int64),
+tcp_s2c_ip_fragments Nullable(Int64),
+tcp_c2s_lost_bytes Nullable(Int64),
+tcp_s2c_lost_bytes Nullable(Int64),
+tcp_c2s_o3_pkts Nullable(Int64),
+tcp_s2c_o3_pkts Nullable(Int64),
+tcp_c2s_rtx_pkts Nullable(Int64),
+tcp_s2c_rtx_pkts Nullable(Int64),
+tcp_c2s_rtx_bytes Nullable(Int64),
+tcp_s2c_rtx_bytes Nullable(Int64),
+tcp_rtt_ms Nullable(Int32),
+tcp_client_isn Nullable(Int64),
+tcp_server_isn Nullable(Int64),
+packet_capture_file String,
+in_src_mac String,
+out_src_mac String,
+in_dest_mac String,
+out_dest_mac String,
+tunnels String,
+dup_traffic_flag Nullable(Int32),
+tunnel_endpoint_a_desc String,
+tunnel_endpoint_b_desc String
+)
+ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand());
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.proxy_event on cluster ck_query (
+recv_time Int64,
+log_id UInt64,
+decoded_as String,
+session_id UInt64,
+start_timestamp_ms DateTime64(3),
+end_timestamp_ms DateTime64(3),
+duration_ms Int32,
+tcp_handshake_latency_ms Nullable(Int32),
+ingestion_time Int64,
+processing_time Int64,
+insert_time Int64,
+device_id String,
+out_link_id Nullable(Int32),
+in_link_id Nullable(Int32),
+device_tag String,
+data_center String,
+device_group String,
+sled_ip String,
+address_type Int32,
+vsys_id Int32,
+t_vsys_id Int32,
+flags Int64,
+flags_identify_info String,
+security_rule_list Array(Int64),
+security_action String,
+monitor_rule_list Array(Int64),
+shaping_rule_list Array(Int64),
+proxy_rule_list Array(Int64),
+statistics_rule_list Array(Int64),
+sc_rule_list Array(Int64),
+sc_rsp_raw Array(Int64),
+sc_rsp_decrypted Array(Int64),
+proxy_action String,
+proxy_pinning_status Nullable(Int32),
+proxy_intercept_status Nullable(Int32),
+proxy_passthrough_reason String,
+proxy_client_side_latency_ms Nullable(Int32),
+proxy_server_side_latency_ms Nullable(Int32),
+proxy_client_side_version String,
+proxy_server_side_version String,
+proxy_cert_verify Nullable(Int32),
+proxy_intercept_error String,
+monitor_mirrored_pkts Nullable(Int32),
+monitor_mirrored_bytes Nullable(Int32),
+client_ip String,
+client_port Int32,
+client_os_desc String,
+client_geolocation LowCardinality(String),
+client_asn Nullable(Int64),
+subscriber_id String,
+imei String,
+imsi String,
+phone_number String,
+apn String,
+server_ip String,
+server_port Int32,
+server_os_desc String,
+server_geolocation LowCardinality(String),
+server_asn Nullable(Int64),
+server_fqdn String,
+server_domain String,
+app_transition String, 
+app LowCardinality(String),
+app_debug_info String,
+app_content String,
+fqdn_category_list Array(Int64),
+ip_protocol LowCardinality(String),
+decoded_path LowCardinality(String),
+http_url String,
+http_host String,
+http_request_line String,
+http_response_line String,
+http_request_body String,
+http_response_body String,
+http_proxy_flag Nullable(Int32),
+http_sequence Nullable(Int32),
+http_cookie String,
+http_referer String,
+http_user_agent String,
+http_request_content_length Nullable(Int64),
+http_request_content_type String,
+http_response_content_length Nullable(Int64),
+http_response_content_type String,
+http_set_cookie String,
+http_version String,
+http_status_code Nullable(Int32),
+http_response_latency_ms Nullable(Int32),
+http_session_duration_ms Nullable(Int32),
+http_action_file_size Nullable(Int64),
+doh_url String,
+doh_host String,
+doh_request_line String,
+doh_response_line String,
+doh_cookie String,
+doh_referer String,
+doh_user_agent String,
+doh_content_length String,
+doh_content_type String,
+doh_set_cookie String,
+doh_version String,
+doh_message_id Int64,
+doh_qr Nullable(Int64),
+doh_opcode Nullable(Int64),
+doh_aa Nullable(Int64),
+doh_tc Nullable(Int64),
+doh_rd Nullable(Int64),
+doh_ra Nullable(Int64),
+doh_rcode Nullable(Int64),
+doh_qdcount Nullable(Int64),
+doh_ancount Nullable(Int64),
+doh_nscount Nullable(Int64),
+doh_arcount Nullable(Int64),
+doh_qname String,
+doh_qtype Nullable(Int64),
+doh_qclass Nullable(Int64),
+doh_cname String,
+doh_sub Nullable(Int64),
+doh_rr String,
+sent_pkts Int64,
+received_pkts Int64,
+sent_bytes Int64,
+received_bytes Int64,
+tcp_c2s_ip_fragments Nullable(Int64),
+tcp_s2c_ip_fragments Nullable(Int64),
+tcp_c2s_lost_bytes Nullable(Int64),
+tcp_s2c_lost_bytes Nullable(Int64),
+tcp_c2s_o3_pkts Nullable(Int64),
+tcp_s2c_o3_pkts Nullable(Int64),
+tcp_c2s_rtx_pkts Nullable(Int64),
+tcp_s2c_rtx_pkts Nullable(Int64),
+tcp_c2s_rtx_bytes Nullable(Int64),
+tcp_s2c_rtx_bytes Nullable(Int64),
+tcp_rtt_ms Nullable(Int32),
+tcp_client_isn Nullable(Int64),
+tcp_server_isn Nullable(Int64),
+packet_capture_file String,
+in_src_mac String,
+out_src_mac String,
+in_dest_mac String,
+out_dest_mac String,
+tunnels String,
+dup_traffic_flag Nullable(Int32),
+tunnel_endpoint_a_desc String,
+tunnel_endpoint_b_desc String
+)
+ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,proxy_event_local,rand());
+
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.security_event_materialized_view on cluster ck_cluster 
+TO tsg_galaxy_v3.security_event_local
+(
+    recv_time Int64,
+    log_id UInt64,
+    decoded_as String,
+    session_id UInt64,
+    start_timestamp_ms DateTime64(3),
+    end_timestamp_ms DateTime64(3),
+    duration_ms Int32,
+    tcp_handshake_latency_ms Nullable(Int32),
+    ingestion_time Int64,
+    processing_time Int64,
+    -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
+    device_id String,
+    out_link_id Nullable(Int32),
+    in_link_id Nullable(Int32),
+    device_tag String,
+    data_center String,
+    device_group String,
+    sled_ip String,
+    address_type Int32,
+    vsys_id Int32,
+    t_vsys_id Int32,
+    flags Int64,
+    flags_identify_info String,
+    security_rule_list Array(Int64),
+    security_action String,
+    monitor_rule_list Array(Int64),
+    shaping_rule_list Array(Int64),
+    proxy_rule_list Array(Int64),
+    statistics_rule_list Array(Int64),
+    sc_rule_list Array(Int64),
+    sc_rsp_raw Array(Int64),
+    sc_rsp_decrypted Array(Int64),
+    proxy_action String,
+    proxy_pinning_status Nullable(Int32),
+    proxy_intercept_status Nullable(Int32),
+    proxy_passthrough_reason String,
+    proxy_client_side_latency_ms Nullable(Int32),
+    proxy_server_side_latency_ms Nullable(Int32),
+    proxy_client_side_version String,
+    proxy_server_side_version String,
+    proxy_cert_verify Nullable(Int32),
+    proxy_intercept_error String,
+    monitor_mirrored_pkts Nullable(Int32),
+    monitor_mirrored_bytes Nullable(Int32),
+    client_ip String,
+    client_port Int32,
+    client_os_desc String,
+    client_geolocation LowCardinality(String),
+    client_asn Nullable(Int64),
+    subscriber_id String,
+    imei String,
+    imsi String,
+    phone_number String,
+    apn String,
+    server_ip String,
+    server_port Int32,
+    server_os_desc String,
+    server_geolocation LowCardinality(String),
+    server_asn Nullable(Int64),
+    server_fqdn String,
+    server_domain String,
+    app_transition String, 
+    app LowCardinality(String),
+    app_debug_info String,
+    app_content String,
+    fqdn_category_list Array(Int64),
+	ip_protocol LowCardinality(String),
+    decoded_path LowCardinality(String),
+    dns_message_id Nullable(Int32),
+    dns_qr Nullable(Int32),
+    dns_opcode Nullable(Int32),
+    dns_aa Nullable(Int32),
+    dns_tc Nullable(Int32),
+    dns_rd Nullable(Int32),
+    dns_ra Nullable(Int32),
+    dns_rcode Nullable(Int32),
+    dns_qdcount Nullable(Int32),
+    dns_ancount Nullable(Int32),
+    dns_nscount Nullable(Int32),
+    dns_arcount Nullable(Int32),
+    dns_qname String,
+    dns_qtype Nullable(Int32),
+    dns_qclass Nullable(Int32),
+    dns_cname String,
+    dns_sub Nullable(Int32),
+    dns_rr String,
+    dns_response_latency_ms Nullable(Int32),
+    http_url String,
+    http_host String,
+    http_request_line String,
+    http_response_line String,
+    http_request_body String,
+    http_response_body String,
+    http_proxy_flag Nullable(Int32),
+    http_sequence Nullable(Int32),
+    http_cookie String,
+    http_referer String,
+    http_user_agent String,
+    http_request_content_length Nullable(Int64),
+    http_request_content_type String,
+    http_response_content_length Nullable(Int64),
+    http_response_content_type String,
+    http_set_cookie String,
+    http_version String,
+    http_status_code Nullable(Int32),
+    http_response_latency_ms Nullable(Int32),
+    http_session_duration_ms Nullable(Int32),
+    http_action_file_size Nullable(Int64),
+    ssl_version String,
+    ssl_sni String,
+    ssl_san String,
+    ssl_cn String,
+    ssl_handshake_latency_ms Nullable(Int32),
+    ssl_ja3_hash String,
+    ssl_ja3s_hash String,
+    ssl_cert_issuer String,
+    ssl_cert_subject String,
+    ssl_esni_flag Nullable(Int32),
+    ssl_ech_flag Nullable(Int32),
+    dtls_cookie String,
+    dtls_version  String,
+    dtls_sni String,
+    dtls_san String,
+    dtls_cn String,
+    dtls_handshake_latency_ms Nullable(Int32),
+    dtls_ja3_fingerprint String,
+    dtls_ja3_hash String,
+    dtls_cert_issuer String,
+    dtls_cert_subject String,
+    mail_protocol_type String,
+    mail_account String,
+    mail_from_cmd String,
+    mail_to_cmd String,
+    mail_from String,
+    mail_password String,
+    mail_to String,
+    mail_cc String,
+    mail_bcc String,
+    mail_subject String,
+    mail_subject_charset String,
+    mail_attachment_name String,
+    mail_attachment_name_charset String,
+    mail_eml_file String,
+    ftp_account String,
+    ftp_url String,
+    ftp_link_type String,
+    quic_version String,
+    quic_sni String,
+    quic_user_agent String,
+    rdp_cookie String,
+    rdp_security_protocol String,
+    rdp_client_channels String,
+    rdp_keyboard_layout String,
+    rdp_client_version String,
+    rdp_client_name String,
+    rdp_client_product_id String,
+    rdp_desktop_width String,
+    rdp_desktop_height String,
+    rdp_requested_color_depth String,
+    rdp_certificate_type String,
+    rdp_certificate_count Nullable(Int32),
+    rdp_certificate_permanent Nullable(Int32),
+    rdp_encryption_level String,
+    rdp_encryption_method String,
+    ssh_version String,
+    ssh_auth_success String,
+    ssh_client_version String,
+    ssh_server_version String,
+    ssh_cipher_alg String,
+    ssh_mac_alg String,
+    ssh_compression_alg String,
+    ssh_kex_alg String,
+    ssh_host_key_alg String,
+    ssh_host_key String,
+    ssh_hassh String,
+    sip_call_id String,
+    sip_originator_description String,
+    sip_responder_description String,
+    sip_user_agent String,
+    sip_server String,
+    sip_originator_sdp_connect_ip String,
+    sip_originator_sdp_media_port Nullable(Int32),
+    sip_originator_sdp_media_type String,
+    sip_originator_sdp_content String,
+    sip_responder_sdp_connect_ip String,
+    sip_responder_sdp_media_port Nullable(Int32),
+    sip_responder_sdp_media_type String,
+    sip_responder_sdp_content String,
+    sip_duration_s Nullable(Int32),
+    sip_bye String,
+    rtp_payload_type_c2s Nullable(Int32),
+    rtp_payload_type_s2c Nullable(Int32),
+    rtp_pcap_path String,
+    rtp_originator_dir Nullable(Int32),
+    stratum_cryptocurrency String,
+    stratum_mining_pools String,
+    stratum_mining_program String,
+    stratum_mining_subscribe String,
+    sent_pkts Int64,
+    received_pkts Int64,
+    sent_bytes Int64,
+    received_bytes Int64,
+    tcp_c2s_ip_fragments Nullable(Int64),
+    tcp_s2c_ip_fragments Nullable(Int64),
+    tcp_c2s_lost_bytes Nullable(Int64),
+    tcp_s2c_lost_bytes Nullable(Int64),
+    tcp_c2s_o3_pkts Nullable(Int64),
+    tcp_s2c_o3_pkts Nullable(Int64),
+    tcp_c2s_rtx_pkts Nullable(Int64),
+    tcp_s2c_rtx_pkts Nullable(Int64),
+    tcp_c2s_rtx_bytes Nullable(Int64),
+    tcp_s2c_rtx_bytes Nullable(Int64),
+    tcp_rtt_ms Nullable(Int32),
+    tcp_client_isn Nullable(Int64),
+    tcp_server_isn Nullable(Int64),
+    packet_capture_file String,
+    in_src_mac String,
+    out_src_mac String,
+    in_dest_mac String,
+    out_dest_mac String,
+    tunnels String,
+    dup_traffic_flag Nullable(Int32),
+    tunnel_endpoint_a_desc String,
+    tunnel_endpoint_b_desc String
+) 
+AS
+SELECT
+    recv_time,
+    log_id,
+    decoded_as,
+    session_id,
+    start_timestamp_ms,
+    end_timestamp_ms,
+    duration_ms,
+    tcp_handshake_latency_ms,
+    ingestion_time,
+    processing_time,
+    -- insert_time,
+    device_id,
+    out_link_id,
+    in_link_id,
+    device_tag,
+    data_center,
+    device_group,
+    sled_ip,
+    address_type,
+    vsys_id,
+    t_vsys_id,
+    flags,
+    flags_identify_info,
+    security_rule_list,
+    security_action,
+    monitor_rule_list,
+    shaping_rule_list,
+    proxy_rule_list,
+    statistics_rule_list,
+    sc_rule_list,
+    sc_rsp_raw,
+    sc_rsp_decrypted,
+    proxy_action,
+    proxy_pinning_status,
+    proxy_intercept_status,
+    proxy_passthrough_reason,
+    proxy_client_side_latency_ms,
+    proxy_server_side_latency_ms,
+    proxy_client_side_version,
+    proxy_server_side_version,
+    proxy_cert_verify,
+    proxy_intercept_error,
+    monitor_mirrored_pkts,
+    monitor_mirrored_bytes,
+    client_ip,
+    client_port,
+    client_os_desc,
+    client_geolocation,
+    client_asn,
+    subscriber_id,
+    imei,
+    imsi,
+    phone_number,
+    apn,
+    server_ip,
+    server_port,
+    server_os_desc,
+    server_geolocation,
+    server_asn,
+    server_fqdn,
+    server_domain,
+    app_transition,
+    app,
+    app_debug_info,
+    app_content,
+    fqdn_category_list,
+    decoded_path,
+    dns_message_id,
+    dns_qr,
+    dns_opcode,
+    dns_aa,
+    dns_tc,
+    dns_rd,
+    dns_ra,
+    dns_rcode,
+    dns_qdcount,
+    dns_ancount,
+    dns_nscount,
+    dns_arcount,
+    dns_qname,
+    dns_qtype,
+    dns_qclass,
+    dns_cname,
+    dns_sub,
+    dns_rr,
+    dns_response_latency_ms,
+    http_url,
+    http_host,
+    http_request_line,
+    http_response_line,
+    http_request_body,
+    http_response_body,
+    http_proxy_flag,
+    http_sequence,
+    http_cookie,
+    http_referer,
+    http_user_agent,
+    http_request_content_length,
+    http_request_content_type,
+    http_response_content_length,
+    http_response_content_type,
+    http_set_cookie,
+    http_version,
+    http_status_code,
+    http_response_latency_ms,
+    http_session_duration_ms,
+    http_action_file_size,
+    ssl_version,
+    ssl_sni,
+    ssl_san,
+    ssl_cn,
+    ssl_handshake_latency_ms,
+    ssl_ja3_hash,
+    ssl_ja3s_hash,
+    ssl_cert_issuer,
+    ssl_cert_subject,
+    ssl_esni_flag,
+    ssl_ech_flag,
+    dtls_cookie,
+    dtls_version,
+    dtls_sni,
+    dtls_san,
+    dtls_cn,
+    dtls_handshake_latency_ms,
+    dtls_ja3_fingerprint,
+    dtls_ja3_hash,
+    dtls_cert_issuer,
+    dtls_cert_subject,
+    mail_protocol_type,
+    mail_account,
+    mail_from_cmd,
+    mail_to_cmd,
+    mail_from,
+    mail_password,
+    mail_to,
+    mail_cc,
+    mail_bcc,
+    mail_subject,
+    mail_subject_charset,
+    mail_attachment_name,
+    mail_attachment_name_charset,
+    mail_eml_file,
+    ftp_account,
+    ftp_url,
+    ftp_link_type,
+    quic_version,
+    quic_sni,
+    quic_user_agent,
+    rdp_cookie,
+    rdp_security_protocol,
+    rdp_client_channels,
+    rdp_keyboard_layout,
+    rdp_client_version,
+    rdp_client_name,
+    rdp_client_product_id,
+    rdp_desktop_width,
+    rdp_desktop_height,
+    rdp_requested_color_depth,
+    rdp_certificate_type,
+    rdp_certificate_count,
+    rdp_certificate_permanent,
+    rdp_encryption_level,
+    rdp_encryption_method,
+    ssh_version,
+    ssh_auth_success,
+    ssh_client_version,
+    ssh_server_version,
+    ssh_cipher_alg,
+    ssh_mac_alg,
+    ssh_compression_alg,
+    ssh_kex_alg,
+    ssh_host_key_alg,
+    ssh_host_key,
+    ssh_hassh,
+    sip_call_id,
+    sip_originator_description,
+    sip_responder_description,
+    sip_user_agent,
+    sip_server,
+    sip_originator_sdp_connect_ip,
+    sip_originator_sdp_media_port,
+    sip_originator_sdp_media_type,
+    sip_originator_sdp_content,
+    sip_responder_sdp_connect_ip,
+    sip_responder_sdp_media_port,
+    sip_responder_sdp_media_type,
+    sip_responder_sdp_content,
+    sip_duration_s,
+    sip_bye,
+    rtp_payload_type_c2s,
+    rtp_payload_type_s2c,
+    rtp_pcap_path,
+    rtp_originator_dir,
+    stratum_cryptocurrency,
+    stratum_mining_pools,
+    stratum_mining_program,
+    stratum_mining_subscribe,
+    sent_pkts,
+    received_pkts,
+    sent_bytes,
+    received_bytes,
+    tcp_c2s_ip_fragments,
+    tcp_s2c_ip_fragments,
+    tcp_c2s_lost_bytes,
+    tcp_s2c_lost_bytes,
+    tcp_c2s_o3_pkts,
+    tcp_s2c_o3_pkts,
+    tcp_c2s_rtx_pkts,
+    tcp_s2c_rtx_pkts,
+    tcp_c2s_rtx_bytes,
+    tcp_s2c_rtx_bytes,
+    tcp_rtt_ms,
+    tcp_client_isn,
+    tcp_server_isn,
+    packet_capture_file,
+    in_src_mac,
+    out_src_mac,
+    in_dest_mac,
+    out_dest_mac,
+    tunnels,
+    dup_traffic_flag,
+    tunnel_endpoint_a_desc,
+    tunnel_endpoint_b_desc
+FROM tsg_galaxy_v3.session_record_local
+WHERE empty(security_rule_list) = 0
+;
+
+-- tsg_galaxy_v3.monitor_event_materialized_view
+CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.monitor_event_materialized_view on cluster ck_cluster 
+TO tsg_galaxy_v3.monitor_event_local
+(
+    recv_time Int64,
+    log_id UInt64,
+    decoded_as String,
+    session_id UInt64,
+    start_timestamp_ms DateTime64(3),
+    end_timestamp_ms DateTime64(3),
+    duration_ms Int32,
+    tcp_handshake_latency_ms Nullable(Int32),
+    ingestion_time Int64,
+    processing_time Int64,
+    -- insert_time Int64 MATERIALIZED toUnixTimestamp(now()),
+    device_id String,
+    out_link_id Nullable(Int32),
+    in_link_id Nullable(Int32),
+    device_tag String,
+    data_center String,
+    device_group String,
+    sled_ip String,
+    address_type Int32,
+    vsys_id Int32,
+    t_vsys_id Int32,
+    flags Int64,
+    flags_identify_info String,
+    security_rule_list Array(Int64),
+    security_action String,
+    monitor_rule_list Array(Int64),
+    shaping_rule_list Array(Int64),
+    proxy_rule_list Array(Int64),
+    statistics_rule_list Array(Int64),
+    sc_rule_list Array(Int64),
+    sc_rsp_raw Array(Int64),
+    sc_rsp_decrypted Array(Int64),
+    proxy_action String,
+    proxy_pinning_status Nullable(Int32),
+    proxy_intercept_status Nullable(Int32),
+    proxy_passthrough_reason String,
+    proxy_client_side_latency_ms Nullable(Int32),
+    proxy_server_side_latency_ms Nullable(Int32),
+    proxy_client_side_version String,
+    proxy_server_side_version String,
+    proxy_cert_verify Nullable(Int32),
+    proxy_intercept_error String,
+    monitor_mirrored_pkts Nullable(Int32),
+    monitor_mirrored_bytes Nullable(Int32),
+    client_ip String,
+    client_port Int32,
+    client_os_desc String,
+    client_geolocation LowCardinality(String),
+    client_asn Nullable(Int64),
+    subscriber_id String,
+    imei String,
+    imsi String,
+    phone_number String,
+    apn String,
+    server_ip String,
+    server_port Int32,
+    server_os_desc String,
+    server_geolocation LowCardinality(String),
+    server_asn Nullable(Int64),
+    server_fqdn String,
+    server_domain String,
+    app_transition String, 
+    app LowCardinality(String),
+    app_debug_info String,
+    app_content String,
+    fqdn_category_list Array(Int64),
+	ip_protocol LowCardinality(String),
+    decoded_path LowCardinality(String),
+    dns_message_id Nullable(Int32),
+    dns_qr Nullable(Int32),
+    dns_opcode Nullable(Int32),
+    dns_aa Nullable(Int32),
+    dns_tc Nullable(Int32),
+    dns_rd Nullable(Int32),
+    dns_ra Nullable(Int32),
+    dns_rcode Nullable(Int32),
+    dns_qdcount Nullable(Int32),
+    dns_ancount Nullable(Int32),
+    dns_nscount Nullable(Int32),
+    dns_arcount Nullable(Int32),
+    dns_qname String,
+    dns_qtype Nullable(Int32),
+    dns_qclass Nullable(Int32),
+    dns_cname String,
+    dns_sub Nullable(Int32),
+    dns_rr String,
+    dns_response_latency_ms Nullable(Int32),
+    http_url String,
+    http_host String,
+    http_request_line String,
+    http_response_line String,
+    http_request_body String,
+    http_response_body String,
+    http_proxy_flag Nullable(Int32),
+    http_sequence Nullable(Int32),
+    http_cookie String,
+    http_referer String,
+    http_user_agent String,
+    http_request_content_length Nullable(Int64),
+    http_request_content_type String,
+    http_response_content_length Nullable(Int64),
+    http_response_content_type String,
+    http_set_cookie String,
+    http_version String,
+    http_status_code Nullable(Int32),
+    http_response_latency_ms Nullable(Int32),
+    http_session_duration_ms Nullable(Int32),
+    http_action_file_size Nullable(Int64),
+    ssl_version String,
+    ssl_sni String,
+    ssl_san String,
+    ssl_cn String,
+    ssl_handshake_latency_ms Nullable(Int32),
+    ssl_ja3_hash String,
+    ssl_ja3s_hash String,
+    ssl_cert_issuer String,
+    ssl_cert_subject String,
+    ssl_esni_flag Nullable(Int32),
+    ssl_ech_flag Nullable(Int32),
+    dtls_cookie String,
+    dtls_version  String,
+    dtls_sni String,
+    dtls_san String,
+    dtls_cn String,
+    dtls_handshake_latency_ms Nullable(Int32),
+    dtls_ja3_fingerprint String,
+    dtls_ja3_hash String,
+    dtls_cert_issuer String,
+    dtls_cert_subject String,
+    mail_protocol_type String,
+    mail_account String,
+    mail_from_cmd String,
+    mail_to_cmd String,
+    mail_from String,
+    mail_password String,
+    mail_to String,
+    mail_cc String,
+    mail_bcc String,
+    mail_subject String,
+    mail_subject_charset String,
+    mail_attachment_name String,
+    mail_attachment_name_charset String,
+    mail_eml_file String,
+    ftp_account String,
+    ftp_url String,
+    ftp_link_type String,
+    quic_version String,
+    quic_sni String,
+    quic_user_agent String,
+    rdp_cookie String,
+    rdp_security_protocol String,
+    rdp_client_channels String,
+    rdp_keyboard_layout String,
+    rdp_client_version String,
+    rdp_client_name String,
+    rdp_client_product_id String,
+    rdp_desktop_width String,
+    rdp_desktop_height String,
+    rdp_requested_color_depth String,
+    rdp_certificate_type String,
+    rdp_certificate_count Nullable(Int32),
+    rdp_certificate_permanent Nullable(Int32),
+    rdp_encryption_level String,
+    rdp_encryption_method String,
+    ssh_version String,
+    ssh_auth_success String,
+    ssh_client_version String,
+    ssh_server_version String,
+    ssh_cipher_alg String,
+    ssh_mac_alg String,
+    ssh_compression_alg String,
+    ssh_kex_alg String,
+    ssh_host_key_alg String,
+    ssh_host_key String,
+    ssh_hassh String,
+    sip_call_id String,
+    sip_originator_description String,
+    sip_responder_description String,
+    sip_user_agent String,
+    sip_server String,
+    sip_originator_sdp_connect_ip String,
+    sip_originator_sdp_media_port Nullable(Int32),
+    sip_originator_sdp_media_type String,
+    sip_originator_sdp_content String,
+    sip_responder_sdp_connect_ip String,
+    sip_responder_sdp_media_port Nullable(Int32),
+    sip_responder_sdp_media_type String,
+    sip_responder_sdp_content String,
+    sip_duration_s Nullable(Int32),
+    sip_bye String,
+    rtp_payload_type_c2s Nullable(Int32),
+    rtp_payload_type_s2c Nullable(Int32),
+    rtp_pcap_path String,
+    rtp_originator_dir Nullable(Int32),
+    stratum_cryptocurrency String,
+    stratum_mining_pools String,
+    stratum_mining_program String,
+    stratum_mining_subscribe String,
+    sent_pkts Int64,
+    received_pkts Int64,
+    sent_bytes Int64,
+    received_bytes Int64,
+    tcp_c2s_ip_fragments Nullable(Int64),
+    tcp_s2c_ip_fragments Nullable(Int64),
+    tcp_c2s_lost_bytes Nullable(Int64),
+    tcp_s2c_lost_bytes Nullable(Int64),
+    tcp_c2s_o3_pkts Nullable(Int64),
+    tcp_s2c_o3_pkts Nullable(Int64),
+    tcp_c2s_rtx_pkts Nullable(Int64),
+    tcp_s2c_rtx_pkts Nullable(Int64),
+    tcp_c2s_rtx_bytes Nullable(Int64),
+    tcp_s2c_rtx_bytes Nullable(Int64),
+    tcp_rtt_ms Nullable(Int32),
+    tcp_client_isn Nullable(Int64),
+    tcp_server_isn Nullable(Int64),
+    packet_capture_file String,
+    in_src_mac String,
+    out_src_mac String,
+    in_dest_mac String,
+    out_dest_mac String,
+    tunnels String,
+    dup_traffic_flag Nullable(Int32),
+    tunnel_endpoint_a_desc String,
+    tunnel_endpoint_b_desc String
+) 
+AS
+SELECT
+    recv_time,
+    log_id,
+    decoded_as,
+    session_id,
+    start_timestamp_ms,
+    end_timestamp_ms,
+    duration_ms,
+    tcp_handshake_latency_ms,
+    ingestion_time,
+    processing_time,
+    -- insert_time,
+    device_id,
+    out_link_id,
+    in_link_id,
+    device_tag,
+    data_center,
+    device_group,
+    sled_ip,
+    address_type,
+    vsys_id,
+    t_vsys_id,
+    flags,
+    flags_identify_info,
+    security_rule_list,
+    security_action,
+    monitor_rule_list,
+    shaping_rule_list,
+    proxy_rule_list,
+    statistics_rule_list,
+    sc_rule_list,
+    sc_rsp_raw,
+    sc_rsp_decrypted,
+    proxy_action,
+    proxy_pinning_status,
+    proxy_intercept_status,
+    proxy_passthrough_reason,
+    proxy_client_side_latency_ms,
+    proxy_server_side_latency_ms,
+    proxy_client_side_version,
+    proxy_server_side_version,
+    proxy_cert_verify,
+    proxy_intercept_error,
+    monitor_mirrored_pkts,
+    monitor_mirrored_bytes,
+    client_ip,
+    client_port,
+    client_os_desc,
+    client_geolocation,
+    client_asn,
+    subscriber_id,
+    imei,
+    imsi,
+    phone_number,
+    apn,
+    server_ip,
+    server_port,
+    server_os_desc,
+    server_geolocation,
+    server_asn,
+    server_fqdn,
+    server_domain,
+    app_transition,
+    app,
+    app_debug_info,
+    app_content,
+    fqdn_category_list,
+    decoded_path,
+    dns_message_id,
+    dns_qr,
+    dns_opcode,
+    dns_aa,
+    dns_tc,
+    dns_rd,
+    dns_ra,
+    dns_rcode,
+    dns_qdcount,
+    dns_ancount,
+    dns_nscount,
+    dns_arcount,
+    dns_qname,
+    dns_qtype,
+    dns_qclass,
+    dns_cname,
+    dns_sub,
+    dns_rr,
+    dns_response_latency_ms,
+    http_url,
+    http_host,
+    http_request_line,
+    http_response_line,
+    http_request_body,
+    http_response_body,
+    http_proxy_flag,
+    http_sequence,
+    http_cookie,
+    http_referer,
+    http_user_agent,
+    http_request_content_length,
+    http_request_content_type,
+    http_response_content_length,
+    http_response_content_type,
+    http_set_cookie,
+    http_version,
+    http_status_code,
+    http_response_latency_ms,
+    http_session_duration_ms,
+    http_action_file_size,
+    ssl_version,
+    ssl_sni,
+    ssl_san,
+    ssl_cn,
+    ssl_handshake_latency_ms,
+    ssl_ja3_hash,
+    ssl_ja3s_hash,
+    ssl_cert_issuer,
+    ssl_cert_subject,
+    ssl_esni_flag,
+    ssl_ech_flag,
+    dtls_cookie,
+    dtls_version,
+    dtls_sni,
+    dtls_san,
+    dtls_cn,
+    dtls_handshake_latency_ms,
+    dtls_ja3_fingerprint,
+    dtls_ja3_hash,
+    dtls_cert_issuer,
+    dtls_cert_subject,
+    mail_protocol_type,
+    mail_account,
+    mail_from_cmd,
+    mail_to_cmd,
+    mail_from,
+    mail_password,
+    mail_to,
+    mail_cc,
+    mail_bcc,
+    mail_subject,
+    mail_subject_charset,
+    mail_attachment_name,
+    mail_attachment_name_charset,
+    mail_eml_file,
+    ftp_account,
+    ftp_url,
+    ftp_link_type,
+    quic_version,
+    quic_sni,
+    quic_user_agent,
+    rdp_cookie,
+    rdp_security_protocol,
+    rdp_client_channels,
+    rdp_keyboard_layout,
+    rdp_client_version,
+    rdp_client_name,
+    rdp_client_product_id,
+    rdp_desktop_width,
+    rdp_desktop_height,
+    rdp_requested_color_depth,
+    rdp_certificate_type,
+    rdp_certificate_count,
+    rdp_certificate_permanent,
+    rdp_encryption_level,
+    rdp_encryption_method,
+    ssh_version,
+    ssh_auth_success,
+    ssh_client_version,
+    ssh_server_version,
+    ssh_cipher_alg,
+    ssh_mac_alg,
+    ssh_compression_alg,
+    ssh_kex_alg,
+    ssh_host_key_alg,
+    ssh_host_key,
+    ssh_hassh,
+    sip_call_id,
+    sip_originator_description,
+    sip_responder_description,
+    sip_user_agent,
+    sip_server,
+    sip_originator_sdp_connect_ip,
+    sip_originator_sdp_media_port,
+    sip_originator_sdp_media_type,
+    sip_originator_sdp_content,
+    sip_responder_sdp_connect_ip,
+    sip_responder_sdp_media_port,
+    sip_responder_sdp_media_type,
+    sip_responder_sdp_content,
+    sip_duration_s,
+    sip_bye,
+    rtp_payload_type_c2s,
+    rtp_payload_type_s2c,
+    rtp_pcap_path,
+    rtp_originator_dir,
+    stratum_cryptocurrency,
+    stratum_mining_pools,
+    stratum_mining_program,
+    stratum_mining_subscribe,
+    sent_pkts,
+    received_pkts,
+    sent_bytes,
+    received_bytes,
+    tcp_c2s_ip_fragments,
+    tcp_s2c_ip_fragments,
+    tcp_c2s_lost_bytes,
+    tcp_s2c_lost_bytes,
+    tcp_c2s_o3_pkts,
+    tcp_s2c_o3_pkts,
+    tcp_c2s_rtx_pkts,
+    tcp_s2c_rtx_pkts,
+    tcp_c2s_rtx_bytes,
+    tcp_s2c_rtx_bytes,
+    tcp_rtt_ms,
+    tcp_client_isn,
+    tcp_server_isn,
+    packet_capture_file,
+    in_src_mac,
+    out_src_mac,
+    in_dest_mac,
+    out_dest_mac,
+    tunnels,
+    dup_traffic_flag,
+    tunnel_endpoint_a_desc,
+    tunnel_endpoint_b_desc
+FROM tsg_galaxy_v3.session_record_local
+WHERE empty(monitor_rule_list) = 0
+;
diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/system.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/system.sql
new file mode 100644
index 0000000..1f45639
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/system.sql
@@ -0,0 +1,9 @@
+create table IF NOT EXISTS `system`.tables_cluster ON CLUSTER ck_query as `system`.tables  ENGINE =Distributed(ck_all,`system`,tables,rand());
+create table IF NOT EXISTS `system`.disks_cluster ON CLUSTER ck_query as `system`.disks  ENGINE =Distributed(ck_all,`system`,disks,rand());
+create table IF NOT EXISTS `system`.parts_cluster ON CLUSTER ck_query as `system`.parts  ENGINE =Distributed(ck_all,`system`,parts,rand());
+create table IF NOT EXISTS `system`.query_log_cluster ON CLUSTER ck_query as `system`.query_log  ENGINE =Distributed(ck_all,`system`,query_log,rand());
+CREATE TABLE IF NOT EXISTS `system`.columns_cluster ON CLUSTER ck_query AS `system`.columns ENGINE=Distributed(ck_all,`system`,columns,rand());
+CREATE TABLE IF NOT EXISTS `system`.processes_cluster ON CLUSTER ck_query AS `system`.processes ENGINE=Distributed(ck_all,`system`,processes,rand());
+
+alter table system.query_log on cluster ck_cluster modify TTL event_date + INTERVAL 60 DAY;
+alter table system.query_log on cluster ck_query modify TTL event_date + INTERVAL 60 DAY;
diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.01-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.01-ck.sql
new file mode 100644
index 0000000..5e7ba07
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.01-ck.sql
@@ -0,0 +1,247 @@
+set distributed_ddl_task_timeout = 180;
+
+ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_app_full_path  String after common_app_id;
+
+ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_app_full_path  String after common_app_id;
+
+ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_app_full_path  String after common_app_id;
+
+ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_app_full_path  String after common_app_id;
+
+ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_app_full_path  String after common_app_id;
+
+ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_app_full_path  String after common_app_id;
+
+ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_app_full_path  String after common_app_id;
+
+ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_app_full_path  String after common_app_id;
+
+ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_app_full_path String after common_app_id;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_app_full_path  String after common_app_id;
+
+
+ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_server_domain  String after common_server_asn;
+
+ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_server_domain  String after common_server_asn;
+
+ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_server_domain  String after common_server_asn;
+
+ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_server_domain  String after common_server_asn;
+
+ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_server_domain  String after common_server_asn;
+
+ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_server_domain  String after common_server_asn;
+
+ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_server_domain  String after common_server_asn;
+
+ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_server_domain  String after common_server_asn;
+
+ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_server_asn;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_server_domain  String after common_server_asn;
+
+
+ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn  String after common_server_asn;
+
+ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn  String after common_server_asn;
+
+ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn  String after common_server_asn;
+
+ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_server_fqdn  String after common_server_asn;
+
+ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_server_fqdn  String after common_server_asn;
+
+ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn  String after common_server_asn;
+
+ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn  String after common_server_asn;
+
+ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_server_fqdn  String after common_server_asn;
+
+ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_server_fqdn String after common_server_asn;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_server_fqdn  String after common_server_asn;
+
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain_local ON CLUSTER ck_cluster(
+	common_log_id UInt64,
+	common_recv_time Int64,
+	common_server_ip String,
+	common_client_ip String,
+	common_sled_ip String,
+	common_entrance_id Int64,
+	common_subscriber_id String,
+	common_stream_trace_id UInt64,
+	common_server_domain String,
+	http_domain String,
+	ssl_sni  String,
+	common_schema_type LowCardinality(String),
+	common_vsys_id Int64 DEFAULT 1,
+	common_client_port Int64,
+	common_server_port Int64,
+	common_app_label LowCardinality(String),
+	common_direction Nullable(Int64),
+	common_data_center String,
+	common_device_group String
+)
+ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(common_recv_time))
+ORDER BY (common_server_domain,common_recv_time);
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_cluster(
+	common_log_id UInt64,
+	common_recv_time Int64,
+	common_server_ip String,
+	common_client_ip String,
+	common_sled_ip String,
+	common_entrance_id Int64,
+	common_subscriber_id String,
+	common_stream_trace_id UInt64,
+	common_server_domain String,
+	http_domain String,
+	ssl_sni  String,
+	common_schema_type LowCardinality(String),
+	common_vsys_id Int64,
+	common_client_port Int64,
+	common_server_port Int64,
+	common_app_label LowCardinality(String),
+	common_direction Nullable(Int64),
+	common_data_center String,
+	common_device_group String
+) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_domain_local,rand());
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy_v3.session_record_common_server_domain ON CLUSTER ck_query(
+	common_log_id UInt64,
+	common_recv_time Int64,
+	common_server_ip String,
+	common_client_ip String,
+	common_sled_ip String,
+	common_entrance_id Int64,
+	common_subscriber_id String,
+	common_stream_trace_id UInt64,
+	common_server_domain String,
+	http_domain String,
+	ssl_sni  String,
+	common_schema_type LowCardinality(String),
+	common_vsys_id Int64,
+	common_client_port Int64,
+	common_server_port Int64,
+	common_app_label LowCardinality(String),
+	common_direction Nullable(Int64),
+	common_data_center String,
+	common_device_group String
+) ENGINE =Distributed(ck_cluster,tsg_galaxy_v3,session_record_common_server_domain_local,rand());
+
+
+ALTER table tsg_galaxy_v3.session_record_common_client_ip_local on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction;
+ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction;
+ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_query add column IF NOT EXISTS common_device_group String after common_direction;
+
+ALTER table tsg_galaxy_v3.session_record_common_server_ip_local on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction;
+ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction;
+ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_query add column IF NOT EXISTS common_device_group String after common_direction;
+
+ALTER table tsg_galaxy_v3.session_record_http_domain_local on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction;
+ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_cluster add column IF NOT EXISTS common_device_group String after common_direction;
+ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_query add column IF NOT EXISTS common_device_group String after common_direction;
+
+
+ALTER table tsg_galaxy_v3.session_record_common_client_ip_local on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction;
+ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction;
+ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_query add column IF NOT EXISTS common_data_center  String after common_direction;
+
+ALTER table tsg_galaxy_v3.session_record_common_server_ip_local on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction;
+ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction;
+ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_query add column IF NOT EXISTS common_data_center  String after common_direction;
+
+ALTER table tsg_galaxy_v3.session_record_http_domain_local on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction;
+ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_cluster add column IF NOT EXISTS common_data_center String after common_direction;
+ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_query add column IF NOT EXISTS common_data_center String after common_direction;
+
+
+ALTER table tsg_galaxy_v3.session_record_common_client_ip_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id;
+ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id;
+ALTER table tsg_galaxy_v3.session_record_common_client_ip on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_stream_trace_id;
+
+ALTER table tsg_galaxy_v3.session_record_common_server_ip_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id;
+ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id;
+ALTER table tsg_galaxy_v3.session_record_common_server_ip on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_stream_trace_id;
+
+ALTER table tsg_galaxy_v3.session_record_http_domain_local on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id;
+ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_cluster add column IF NOT EXISTS common_server_domain String after common_stream_trace_id;
+ALTER table tsg_galaxy_v3.session_record_http_domain on cluster ck_query add column IF NOT EXISTS common_server_domain String after common_stream_trace_id;
+
+
+drop view IF EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster;
+drop view IF EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster;
+drop view IF EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster;
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_client_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_client_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local;
+CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_http_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_http_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local;
+CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_ip ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_ip_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local;
+CREATE MATERIALIZED VIEW IF NOT EXISTS tsg_galaxy_v3.common_server_domain ON CLUSTER ck_cluster TO tsg_galaxy_v3.session_record_common_server_domain_local AS SELECT common_log_id, common_recv_time, common_server_ip, common_client_ip, common_sled_ip, common_entrance_id, common_subscriber_id, common_stream_trace_id, common_server_domain, http_domain, ssl_sni, common_schema_type, common_vsys_id, common_client_port, common_server_port, common_app_label, common_direction,common_data_center, common_device_group FROM tsg_galaxy_v3.session_record_local;
+
+drop table  IF  EXISTS `system`.query_log_cluster on cluster ck_query;
+drop table  IF  EXISTS `system`.disks_cluster on cluster ck_query;
+drop table  IF  EXISTS `system`.columns_cluster on cluster ck_query;
+drop table  IF  EXISTS `system`.parts_cluster on cluster ck_query;
+drop table  IF  EXISTS `system`.processes_cluster on cluster ck_query;
+drop table  IF  EXISTS `system`.tables_cluster on cluster ck_query;
+
+
+create table IF NOT EXISTS `system`.tables_cluster ON CLUSTER ck_query as `system`.tables  ENGINE =Distributed(ck_all,`system`,tables,rand());
+create table IF NOT EXISTS `system`.disks_cluster ON CLUSTER ck_query as `system`.disks  ENGINE =Distributed(ck_all,`system`,disks,rand());
+create table IF NOT EXISTS `system`.parts_cluster ON CLUSTER ck_query as `system`.parts  ENGINE =Distributed(ck_all,`system`,parts,rand());
+create table IF NOT EXISTS `system`.query_log_cluster ON CLUSTER ck_query as `system`.query_log  ENGINE =Distributed(ck_all,`system`,query_log,rand());
+CREATE TABLE IF NOT EXISTS `system`.columns_cluster ON CLUSTER ck_query AS `system`.columns ENGINE=Distributed(ck_all,`system`,columns,rand());
+CREATE TABLE IF NOT EXISTS `system`.processes_cluster ON CLUSTER ck_query AS `system`.processes ENGINE=Distributed(ck_all,`system`,processes,rand());
+alter table system.query_log on cluster ck_cluster modify TTL event_date + INTERVAL 60 DAY;
+alter table system.query_log on cluster ck_query modify TTL event_date + INTERVAL 60 DAY;
+
diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.02-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.02-ck.sql
new file mode 100644
index 0000000..042b51f
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.02-ck.sql
@@ -0,0 +1,110 @@
+set distributed_ddl_task_timeout = 180;
+
+ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS http_request_body_key  String after common_flags;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS http_request_body_key  String after common_flags;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS http_request_body_key  String after common_flags;
+
+ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+
+
+
+ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+
+
+
+ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+
+
+ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+
+
+ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+ 
+
+ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+
+
+ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+
+
+
+ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_flags_identify_info  String after common_flags;
+
+
+ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+
+ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+
+
+
+ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+
+
+
+ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+
+
+ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+
+
+ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ 
+
+ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+
+
+ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+
+
+ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_shaping_rule_ids  Array(Int64) after common_user_region;
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.03-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.03-ck.sql
new file mode 100644
index 0000000..1d3d958
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.03-ck.sql
@@ -0,0 +1,47 @@
+set distributed_ddl_task_timeout = 180;
+
+alter table system.query_log on cluster ck_cluster modify TTL event_date + INTERVAL 60 DAY;
+
+
+alter table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time;
+alter table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+alter table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+
+
+alter table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time;
+alter table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+alter table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+
+
+alter table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time;
+alter table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+alter table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+
+alter table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time;
+alter table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+alter table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+
+alter table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time;
+alter table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+alter table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+
+alter table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time;
+alter table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+alter table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+
+alter table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time;
+alter table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+alter table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+
+alter table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time;
+alter table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+alter table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+
+alter table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64 MATERIALIZED toUnixTimestamp(now()) after common_processing_time;
+alter table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+alter table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_insert_time Int64  after common_processing_time;
+
+
+
+
+
diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.04-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.04-ck.sql
new file mode 100644
index 0000000..0e50ac4
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.04-ck.sql
@@ -0,0 +1,6 @@
+set distributed_ddl_task_timeout = 180;
+
+
+ALTER table tsg_galaxy_v3.dos_event_local on cluster ck_cluster add column IF NOT EXISTS profile_id UInt64 after log_id;
+ALTER table tsg_galaxy_v3.dos_event on cluster ck_cluster add column IF NOT EXISTS profile_id UInt64 after log_id;
+ALTER table tsg_galaxy_v3.dos_event on cluster ck_query add column IF NOT EXISTS profile_id UInt64 after log_id;
\ No newline at end of file
diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.05-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.05-ck.sql
new file mode 100644
index 0000000..769c47e
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.05-ck.sql
@@ -0,0 +1,11 @@
+set distributed_ddl_task_timeout = 180;
+
+
+ALTER  table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie;
+
+
+ALTER TABLE tsg_galaxy_v3.proxy_event_local on cluster ck_cluster DROP COLUMN IF  EXISTS rdp_encryption_method, DROP COLUMN IF  EXISTS rdp_encryption_level, DROP COLUMN IF  EXISTS rdp_certificate_permanent, DROP COLUMN IF  EXISTS rdp_certificate_count, DROP COLUMN IF  EXISTS rdp_certificate_type, DROP COLUMN IF  EXISTS rdp_requested_color_depth, DROP COLUMN IF  EXISTS rdp_desktop_height, DROP COLUMN IF  EXISTS rdp_desktop_width, DROP COLUMN IF  EXISTS rdp_client_product_id, DROP COLUMN IF  EXISTS rdp_client_name, DROP COLUMN IF  EXISTS rdp_client_version, DROP COLUMN IF  EXISTS rdp_keyboard_layout, DROP COLUMN IF  EXISTS rdp_client_channels, DROP COLUMN IF  EXISTS rdp_security_protocol, DROP COLUMN IF  EXISTS rdp_cookie;
+ALTER TABLE tsg_galaxy_v3.proxy_event on cluster ck_cluster DROP COLUMN IF  EXISTS rdp_encryption_method, DROP COLUMN IF  EXISTS rdp_encryption_level, DROP COLUMN IF  EXISTS rdp_certificate_permanent, DROP COLUMN IF  EXISTS rdp_certificate_count, DROP COLUMN IF  EXISTS rdp_certificate_type, DROP COLUMN IF  EXISTS rdp_requested_color_depth, DROP COLUMN IF  EXISTS rdp_desktop_height, DROP COLUMN IF  EXISTS rdp_desktop_width, DROP COLUMN IF  EXISTS rdp_client_product_id, DROP COLUMN IF  EXISTS rdp_client_name, DROP COLUMN IF  EXISTS rdp_client_version, DROP COLUMN IF  EXISTS rdp_keyboard_layout, DROP COLUMN IF  EXISTS rdp_client_channels, DROP COLUMN IF  EXISTS rdp_security_protocol, DROP COLUMN IF  EXISTS rdp_cookie;
+ALTER TABLE tsg_galaxy_v3.proxy_event on cluster ck_query DROP COLUMN IF  EXISTS rdp_encryption_method, DROP COLUMN IF  EXISTS rdp_encryption_level, DROP COLUMN IF  EXISTS rdp_certificate_permanent, DROP COLUMN IF  EXISTS rdp_certificate_count, DROP COLUMN IF  EXISTS rdp_certificate_type, DROP COLUMN IF  EXISTS rdp_requested_color_depth, DROP COLUMN IF  EXISTS rdp_desktop_height, DROP COLUMN IF  EXISTS rdp_desktop_width, DROP COLUMN IF  EXISTS rdp_client_product_id, DROP COLUMN IF  EXISTS rdp_client_name, DROP COLUMN IF  EXISTS rdp_client_version, DROP COLUMN IF  EXISTS rdp_keyboard_layout, DROP COLUMN IF  EXISTS rdp_client_channels, DROP COLUMN IF  EXISTS rdp_security_protocol, DROP COLUMN IF  EXISTS rdp_cookie;
diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.06-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.06-ck.sql
new file mode 100644
index 0000000..11c06a8
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.06-ck.sql
@@ -0,0 +1,159 @@
+set distributed_ddl_task_timeout = 180;
+
+
+ALTER  table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS intercept_error String after rdp_cookie,add column IF NOT EXISTS intercept_cert_verify Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_client_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_server_side_version String after rdp_cookie,add column IF NOT EXISTS intercept_client_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_server_side_latency Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_passthrough_reason String after rdp_cookie,add column IF NOT EXISTS intercept_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS intercept_pinning_status Nullable(Int64) after rdp_cookie,add column IF NOT EXISTS ssl_cert_subject String after rdp_cookie,add column IF NOT EXISTS ssl_cert_issuer String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3s_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_hash String after rdp_cookie,add column IF NOT EXISTS ssl_ja3_fingerprint String after rdp_cookie,add column IF NOT EXISTS ssl_con_latency_ms Int64 after rdp_cookie,add column IF NOT EXISTS ssl_cn String after rdp_cookie,add column IF NOT EXISTS ssl_san String after rdp_cookie,add column IF NOT EXISTS ssl_sni String after rdp_cookie,add column IF NOT EXISTS ssl_version String after rdp_cookie;
+
+
+ALTER TABLE tsg_galaxy_v3.proxy_event_local on cluster ck_cluster DROP COLUMN IF  EXISTS rdp_encryption_method, DROP COLUMN IF  EXISTS rdp_encryption_level, DROP COLUMN IF  EXISTS rdp_certificate_permanent, DROP COLUMN IF  EXISTS rdp_certificate_count, DROP COLUMN IF  EXISTS rdp_certificate_type, DROP COLUMN IF  EXISTS rdp_requested_color_depth, DROP COLUMN IF  EXISTS rdp_desktop_height, DROP COLUMN IF  EXISTS rdp_desktop_width, DROP COLUMN IF  EXISTS rdp_client_product_id, DROP COLUMN IF  EXISTS rdp_client_name, DROP COLUMN IF  EXISTS rdp_client_version, DROP COLUMN IF  EXISTS rdp_keyboard_layout, DROP COLUMN IF  EXISTS rdp_client_channels, DROP COLUMN IF  EXISTS rdp_security_protocol, DROP COLUMN IF  EXISTS rdp_cookie;
+ALTER TABLE tsg_galaxy_v3.proxy_event on cluster ck_cluster DROP COLUMN IF  EXISTS rdp_encryption_method, DROP COLUMN IF  EXISTS rdp_encryption_level, DROP COLUMN IF  EXISTS rdp_certificate_permanent, DROP COLUMN IF  EXISTS rdp_certificate_count, DROP COLUMN IF  EXISTS rdp_certificate_type, DROP COLUMN IF  EXISTS rdp_requested_color_depth, DROP COLUMN IF  EXISTS rdp_desktop_height, DROP COLUMN IF  EXISTS rdp_desktop_width, DROP COLUMN IF  EXISTS rdp_client_product_id, DROP COLUMN IF  EXISTS rdp_client_name, DROP COLUMN IF  EXISTS rdp_client_version, DROP COLUMN IF  EXISTS rdp_keyboard_layout, DROP COLUMN IF  EXISTS rdp_client_channels, DROP COLUMN IF  EXISTS rdp_security_protocol, DROP COLUMN IF  EXISTS rdp_cookie;
+ALTER TABLE tsg_galaxy_v3.proxy_event on cluster ck_query DROP COLUMN IF  EXISTS rdp_encryption_method, DROP COLUMN IF  EXISTS rdp_encryption_level, DROP COLUMN IF  EXISTS rdp_certificate_permanent, DROP COLUMN IF  EXISTS rdp_certificate_count, DROP COLUMN IF  EXISTS rdp_certificate_type, DROP COLUMN IF  EXISTS rdp_requested_color_depth, DROP COLUMN IF  EXISTS rdp_desktop_height, DROP COLUMN IF  EXISTS rdp_desktop_width, DROP COLUMN IF  EXISTS rdp_client_product_id, DROP COLUMN IF  EXISTS rdp_client_name, DROP COLUMN IF  EXISTS rdp_client_version, DROP COLUMN IF  EXISTS rdp_keyboard_layout, DROP COLUMN IF  EXISTS rdp_client_channels, DROP COLUMN IF  EXISTS rdp_security_protocol, DROP COLUMN IF  EXISTS rdp_cookie;
+
+
+ALTER  table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+
+ALTER  table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+
+ALTER  table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+
+ALTER  table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+
+ALTER  table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+
+ALTER  table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+
+ALTER  table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+
+ALTER  table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_out_dest_mac  String after common_server_domain;
+
+
+ALTER  table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+
+ALTER  table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+
+ALTER  table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+
+ALTER  table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+
+ALTER  table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+
+ALTER  table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+
+ALTER  table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+
+ALTER  table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+ALTER  table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_in_dest_mac  String after common_server_domain;
+
+
+ALTER  table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+
+ALTER  table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+
+ALTER  table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+
+ALTER  table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+
+ALTER  table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+
+ALTER  table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+
+ALTER  table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+
+ALTER  table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_out_src_mac  String after common_phone_number;
+
+
+ALTER  table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+
+ALTER  table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+
+ALTER  table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+
+ALTER  table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+
+ALTER  table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+
+ALTER  table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+
+ALTER  table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+
+ALTER  table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
+ALTER  table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_in_src_mac  String after common_phone_number;
diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.07-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.07-ck.sql
new file mode 100644
index 0000000..27bf30b
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.07-ck.sql
@@ -0,0 +1,97 @@
+set distributed_ddl_task_timeout = 180;
+
+
+ALTER  table tsg_galaxy_v3.session_record_local on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_query drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type;
+
+ALTER  table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_query drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type;
+
+ALTER  table tsg_galaxy_v3.security_event_local on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_cluster drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_query drop column IF EXISTS voip_calling_account, drop column IF EXISTS voip_called_account, drop column IF EXISTS voip_calling_number, drop column IF EXISTS voip_called_number, drop column IF EXISTS streaming_media_url, drop column IF EXISTS streaming_media_protocol, drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type;
+
+ALTER  table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_cluster drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_query drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type;
+
+ALTER  table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type;
+ALTER  table tsg_galaxy_v3.transaction_record on cluster ck_cluster drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type;
+ALTER  table tsg_galaxy_v3.transaction_record on cluster ck_query drop column IF EXISTS http_request_body_key, drop column IF EXISTS http_response_body_key, drop column IF EXISTS http_content_length, drop column IF EXISTS http_content_type;
+
+
+ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+
+
+ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
\ No newline at end of file
diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.08-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.08-ck.sql
new file mode 100644
index 0000000..df1ba02
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.08-ck.sql
@@ -0,0 +1,158 @@
+set distributed_ddl_task_timeout = 180;
+
+
+ALTER  table tsg_galaxy_v3.session_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+
+ALTER  table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+
+ALTER  table tsg_galaxy_v3.security_event_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+
+ALTER  table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+
+ALTER  table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.transaction_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.transaction_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+
+ALTER  table tsg_galaxy_v3.radius_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.radius_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.radius_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+
+ALTER  table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.gtpc_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.gtpc_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+
+ALTER  table tsg_galaxy_v3.voip_record_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.voip_record on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.voip_record on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+
+ALTER  table tsg_galaxy_v3.active_defence_event_local on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.active_defence_event on cluster ck_cluster rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+ALTER  table tsg_galaxy_v3.active_defence_event on cluster ck_query rename column IF EXISTS common_egress_link_id TO common_out_link_id;
+
+
+ALTER  table tsg_galaxy_v3.session_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+
+ALTER  table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+
+ALTER  table tsg_galaxy_v3.security_event_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+
+ALTER  table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+
+ALTER  table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.transaction_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.transaction_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+
+ALTER  table tsg_galaxy_v3.radius_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.radius_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.radius_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+
+ALTER  table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.gtpc_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.gtpc_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+
+ALTER  table tsg_galaxy_v3.voip_record_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.voip_record on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.voip_record on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+
+ALTER  table tsg_galaxy_v3.active_defence_event_local on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.active_defence_event on cluster ck_cluster rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+ALTER  table tsg_galaxy_v3.active_defence_event on cluster ck_query rename column IF EXISTS common_ingress_link_id TO common_in_link_id;
+
+
+ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+
+
+ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
\ No newline at end of file
diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.09-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.09-ck.sql
new file mode 100644
index 0000000..1528e0c
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.09-ck.sql
@@ -0,0 +1,131 @@
+set distributed_ddl_task_timeout = 180;
+
+
+ALTER  table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version;
+
+ALTER  table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version;
+
+ALTER  table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version;
+
+ALTER  table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version;
+
+ALTER  table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version;
+ALTER  table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS http_status_code Int64 after http_version;
+ALTER  table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS http_status_code Int64 after http_version;
+
+
+ALTER  table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject;
+
+ALTER  table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject;
+
+ALTER  table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject;
+
+ALTER  table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS ssl_ech_flag Int64 after ssl_cert_subject;
+
+
+ALTER  table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject;
+ALTER  table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject;
+
+ALTER  table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject;
+ALTER  table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject;
+
+ALTER  table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject;
+ALTER  table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject;
+
+ALTER  table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject;
+ALTER  table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS ssl_esni_flag Int64 after ssl_cert_subject;
+
+
+ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_start_timestamp_ms Datetime64(3) after common_end_time;
+
+
+
+ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+
+ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_end_timestamp_ms Datetime64(3) after common_start_timestamp_ms;
diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.10-ck.sql b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.10-ck.sql
new file mode 100644
index 0000000..20b7e1e
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/clickhouse/files/upgrade-sql/update-23.10-ck.sql
@@ -0,0 +1,76 @@
+set distributed_ddl_task_timeout = 180;
+
+ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port;
+
+ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port;
+
+ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port;
+
+ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port;
+
+ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port;
+
+ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port;
+
+ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port;
+
+ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port;
+
+ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_client_os_name String after common_client_port;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_client_os_name String after common_client_port;
+
+
+
+ALTER table tsg_galaxy_v3.session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.session_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port;
+
+ALTER table tsg_galaxy_v3.interim_session_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.interim_session_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port;
+
+ALTER table tsg_galaxy_v3.security_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.security_event on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port;
+
+ALTER table tsg_galaxy_v3.proxy_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.proxy_event on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port;
+
+ALTER table tsg_galaxy_v3.transaction_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.transaction_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port;
+
+ALTER table tsg_galaxy_v3.sys_packet_capture_event_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.sys_packet_capture_event on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port;
+
+ALTER table tsg_galaxy_v3.radius_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.radius_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port;
+
+ALTER table tsg_galaxy_v3.gtpc_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.gtpc_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port;
+
+ALTER table tsg_galaxy_v3.voip_record_local on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_cluster add column IF NOT EXISTS common_server_os_name String after common_server_port;
+ALTER table tsg_galaxy_v3.voip_record on cluster ck_query add column IF NOT EXISTS common_server_os_name String after common_server_port;
+
diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/tasks/init.yml b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/init.yml
new file mode 100644
index 0000000..9c4c8b9
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/init.yml
@@ -0,0 +1,24 @@
+- name: Setting clickhouse init directory
+  set_fact:
+   init_path: '{{ deploy_dir }}/clickhouse/init/'
+
+- name: Creatting clickhouse init directory
+  file:
+    path: '{{ init_path }}'
+    state: directory
+
+- name: Copying Clickhouse create tables sql
+  copy:
+    src: 'files/{{ item }}'
+    dest: '{{ init_path }}/{{ item }}'
+    force: true
+  with_items:
+    - create_ck_table.sql
+    - system.sql
+
+- name: Creating tables
+  shell: clickhouse-client -h {{ inventory_hostname }} --port 9001 -m -u default --password {{ clickhouse_default_pin }} --multiquery < {{ init_path }}/{{ item }}
+  loop:
+    - create_ck_table.sql
+    - system.sql
+  run_once: true
diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/tasks/main.yml b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/main.yml
new file mode 100644
index 0000000..951be61
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/main.yml
@@ -0,0 +1,5 @@
+- include: init.yml
+  when: inventory_hostname in groups['clickhouse'][:2] and operation == "install"
+
+- include: upgrate.yml
+  when: inventory_hostname in groups['clickhouse'][:2] and operation == "upgrade"
diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/tasks/upgrate.yml b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/upgrate.yml
new file mode 100644
index 0000000..48401ee
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/clickhouse/tasks/upgrate.yml
@@ -0,0 +1,25 @@
+- name: Setting clickhouse upgrade directory
+  set_fact:
+   upgrade_path: '{{ deploy_dir }}/clickhouse/upgrade/'
+
+- name: Creatting clickhouse upgrade directory
+  file:
+    path: '{{ upgrade_path }}'
+    state: directory
+
+- name: Copying clickhouse upgrade tables sql
+  copy:
+    src: 'files/upgrade-sql'
+    dest: '{{ upgrade_path }}/'
+    force: true
+
+- name: Copying clickhouse upgrade script
+  template:
+    src: 'upgrade-clickhouse-data.sh.j2'
+    dest: '{{ upgrade_path }}/upgrade-clickhouse-data.sh'
+    mode: 0075
+    force: true
+
+- name: Upgrade clickhouse data
+  shell: cd {{ upgrade_path }} && sh upgrade-clickhouse-data.sh
+  run_once: true
diff --git a/tsg-olap/parcels/roles/initialization/clickhouse/templates/upgrade-clickhouse-data.sh.j2 b/tsg-olap/parcels/roles/initialization/clickhouse/templates/upgrade-clickhouse-data.sh.j2
new file mode 100644
index 0000000..bc9fd19
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/clickhouse/templates/upgrade-clickhouse-data.sh.j2
@@ -0,0 +1,50 @@
+#!/bin/bash
+source /etc/profile
+set -e
+
+# 数据库连接参数
+upgrade_dir=${1:-$(pwd)/upgrade-sql}
+
+# 指定当前版本和历史版本
+current_version="{{ solution_version }}"
+old_version="{{ old_version }}"
+
+# 数据库连接参数
+db_hostname="127.0.0.1"
+db_username="default"
+db_password="galaxy2019"
+
+suffix=".sql"
+# 获取更新目录中的所有SQL文件
+sql_files=$(find "${upgrade_dir}" -name "*$suffix")
+
+# 标记是否开始执行历史版本的标识
+start_execution=false
+
+# 循环处理每个SQL文件
+for file in ${sql_files}; do
+    # 从文件名中提取版本号
+    filename=$(basename "$file")
+    version=$(echo "$filename" | grep -oE '[0-9]+(\.[0-9]+)?' | tr -d '.')
+    current_version="${current_version//./}"
+    old_version="${old_version//./}"
+    # 检查是否达到历史版本
+    if [[ "${version}" -gt "${old_version}" ]]; then
+        start_execution=true
+    fi
+
+    # 检查是否开始执行历史版本之后的SQL文件
+    if [[ ${start_execution} = true ]]; then
+        echo "Executing SQL file: ${file}"
+
+        # 执行SQL文件
+        clickhouse-client -h 127.0.0.1 --port 9001 -m -u "${db_username}" --password "${db_password}" --multiquery <"${file}"
+        echo "Upgrade ${version} successfully"
+    fi
+
+    # 检查是否达到当前版本
+    if [[ "${version}" -ge "${current_version}" ]]; then
+        break
+    fi
+done
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/backup.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/backup.yml
new file mode 100644
index 0000000..899461b
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/tasks/backup.yml
@@ -0,0 +1,25 @@
+- name: Check if backup directory exists
+  stat:
+    path: "{{ backup_path }}"
+  register: exist_status
+
+- name: Backup directories
+  block:
+    - name: Creat backup directory
+      file:
+        state: directory
+        path: "{{ backup_path }}"
+
+    - name: Copying directories to backup
+      copy:
+        src: "{{ deploy_dir }}/{{ container_name }}/{{ item }}"
+        dest: "{{ backup_path }}"
+        remote_src: true
+      loop: "{{ backup_items }}"
+
+  rescue:
+    - name: Remove backup directory on failure
+      file:
+        path: "{{ backup_path }}"
+        state: absent
+  when: exist_status.stat.exists == false
diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/check.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/check.yml
new file mode 100644
index 0000000..295e895
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/tasks/check.yml
@@ -0,0 +1,23 @@
+- name: Get druid running tasks
+  shell: curl -s http://{{ vrrp_instance.default.virtual_ipaddress }}:8089/druid/indexer/v1/supervisor?state=true | jq -r '.[] | select(.state == "RUNNING") | .id'
+  register: run_task_list
+  run_once: true
+
+- name: Get init task template
+  find:
+    paths: "templates/tasks/"
+    file_type: file
+    patterns: "*.json"
+  register: find_result
+  delegate_to: 127.0.0.1
+  run_once: true
+
+- name: Set init task list
+  set_fact:
+    init_task_list: "{{ find_result.files | map(attribute='path') | map('basename') | map('regex_replace', '^index_kafka_(.*)\\.json$', '\\1') | list }}"
+
+- name: Verify that the number of running tasks is correct
+  fail:
+    msg: "The number of running druid tasks is less than the normal number!"
+  when: run_task_list.stdout_lines | intersect(init_task_list) |length != (init_task_list|length)
+  run_once: true
diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/init-cluster.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/init-cluster.yml
new file mode 100644
index 0000000..aaff071
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/tasks/init-cluster.yml
@@ -0,0 +1,115 @@
+- name: Setting init directory
+  set_fact:
+   init_path: '{{ deploy_dir }}/druid/init/'
+   topology_path: '{{ deploy_dir }}/druid/topology/'
+
+- name: Remove files and directories
+  file:
+    path: "{{ init_path }}"
+    state: absent
+  ignore_errors: true
+  loop:
+    - "{{ init_path }}"
+    - "{{ topology_path }}"
+
+- name: Creatting init directory
+  file:
+    path: '{{ item }}'
+    state: directory
+  loop:
+    - "{{ init_path }}"
+    - "{{ topology_path }}"
+    - "{{ topology_path }}/tasks"
+
+- name: Setting hdfs_ip variable
+  set_fact: 
+      hdfs_ip: "{{groups.hdfs[0]}}"
+
+- name: Unzipping segments.zip
+  unarchive:
+    src: 'files/cluster/segments.zip'
+    dest: '{{ deploy_dir }}/'
+    force: true
+  run_once: true
+  delegate_to: '{{ hdfs_ip }}'
+
+- name: Copying push segments to hdfs installation path
+  copy:
+    src: 'files/cluster/push_segments.sh'
+    dest: '{{ deploy_dir }}/push_segments.sh'
+    force: true
+    mode: 0755
+  run_once: true
+  delegate_to: '{{ hdfs_ip }}'
+
+- name: Creating druid directory and putting segments to hdfs
+  shell: source /etc/profile && hadoop fs -mkdir -p /druid/segments
+  register: nums_out
+  run_once: true
+  delegate_to: '{{ hdfs_ip }}'
+  ignore_errors: yes
+
+- name: Putting segments to hdfs
+  shell:  source /etc/profile && cd {{ deploy_dir }}/ && sh push_segments.sh
+  register: nums_out
+  run_once: true
+  delegate_to: '{{ hdfs_ip }}'
+
+- name: Copying mysql Client to /usr/bin/
+  copy:
+    src: 'files/mysql'
+    dest: '/usr/bin/'
+    mode: 0755
+    force: true
+
+- name: Copying init files to {{ init_path }}
+  copy:
+    src: 'files/cluster/'
+    dest: '{{ deploy_dir }}/druid/init'
+    force: true
+
+- name: Copying tasks template to {{ topology_path }}
+  template:
+    src: "{{ item }}"
+    dest: "{{ deploy_dir }}/druid/topology/tasks/{{ item | basename | regex_replace('.j2$', '') }}"
+    mode: "0644"
+    force: true
+  with_fileglob: "templates/tasks/*"
+
+- name: Inserting segments to mariadb
+  shell: 'mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ vrrp_instance.default.virtual_ipaddress }} druid < {{ init_path }}/druid_segments-tsg.sql'
+  run_once: true
+
+- name: Copying script files
+  template:
+    src: '{{ item }}.j2'
+    dest: '{{ topology_path }}/{{ item }}'
+    mode: 0755
+  with_items:
+    - 'druid_monitor.sh'
+    - 'supervisor-manager.sh'
+
+- name: Copying monitor template
+  template:
+    src: 'run_druid_monitor.j2'
+    dest: '/etc/cron.d/run_druid_monitor'
+    mode: 0644
+
+- name: Submitting druid tasks
+  shell: cd {{ topology_path }} && sh supervisor-manager.sh startall
+  run_once: true
+
+- name: Deleting {{ deploy_dir }}/push_segments.sh
+  file:
+    path: "{{ deploy_dir }}/push_segments.sh"
+    state: absent
+  run_once: true
+  delegate_to: '{{ hdfs_ip }}'
+
+- name: Deleting {{ deploy_dir }}/segments.zip
+  file:
+    path: "{{ deploy_dir }}/segments.zip"
+    state: absent
+  run_once: true
+  delegate_to: '{{ hdfs_ip }}'
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/init-standalone.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/init-standalone.yml
new file mode 100644
index 0000000..9c10b77
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/tasks/init-standalone.yml
@@ -0,0 +1,65 @@
+- name: Setting init directory
+  set_fact:
+   init_path: '{{ deploy_dir }}/druid/init/'
+   topology_path: '{{ deploy_dir }}/druid/topology/'
+
+- name: Creatting init directory
+  file:
+    path: '{{ item }}'
+    state: directory
+  loop:
+    - '{{ init_path }}'
+    - '{{ topology_path }}'
+    - '{{ topology_path }}/tasks'
+    - '{{ deploy_dir }}/druid/var/druid'
+
+- name: Unzipping segments.zip
+  unarchive:
+    src: 'files/standalone/segments.zip'
+    dest: '{{ deploy_dir }}/druid/var/druid/'
+    force: true
+
+- name: Copying mysql Client to /usr/bin/
+  copy:
+    src: 'files/mysql'
+    dest: '/usr/bin/'
+    mode: 0755
+    force: true
+
+- name: Copying init files to {{ init_path }}
+  copy:
+    src: 'files/standalone'
+    dest: '{{ init_path }}/'
+    force: true
+
+- name: Copying tasks template to {{ topology_path }}
+  template:
+    src: "{{ item }}"
+    dest: "{{ deploy_dir }}/druid/topology/tasks/{{ item | basename | regex_replace('.j2$', '') }}"
+    mode: "0644"
+    force: true
+  with_fileglob: "templates/tasks/*"
+
+
+- name: Inserting segments to mariadb
+  shell: 'mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ vrrp_instance.default.virtual_ipaddress }} druid < {{ init_path }}/standalone/druid_segments-tsg.sql'
+  run_once: true
+
+- name: Copying script files
+  template:
+    src: '{{ item }}.j2'
+    dest: '{{ topology_path }}/{{ item }}'
+    mode: 0755
+  with_items:
+    - "druid_monitor.sh"
+    - "supervisor-manager.sh"
+
+- name: Copying monitor template
+  template:
+    src: 'run_druid_monitor.j2'
+    dest: '/etc/cron.d/run_druid_monitor'
+    mode: 0644
+
+- name: Submitting druid tasks
+  shell: cd {{ topology_path }} && sh supervisor-manager.sh startall
+  run_once: true
diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/main.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/main.yml
new file mode 100644
index 0000000..35bf1ff
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/tasks/main.yml
@@ -0,0 +1,21 @@
+- block:
+  - include: uninstall.yml
+  - include: "{{ playbook_name }}"
+    vars: 
+      playbook_name: "{{ 'init-cluster.yml' if groups.druid | length > 1 else 'init-standalone.yml' }}"
+  - include: check.yml
+  when: inventory_hostname in groups['druid'][:2] and operation == 'install'
+
+- block:
+  - include: backup.yml
+  - include: uninstall.yml
+  - include: "{{ playbook_name }}"
+    vars:
+      playbook_name: "{{ 'init-cluster.yml' if groups.druid | length > 1 else 'init-standalone.yml' }}"
+  - include: check.yml
+  when: inventory_hostname in groups['druid'][:2] and (operation) == "upgrade"
+
+- block:
+  - include: uninstall.yml
+  when: inventory_hostname in groups['druid'][:2] and (operation) == "uninstall"
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/tasks/uninstall.yml b/tsg-olap/parcels/roles/initialization/druid/tasks/uninstall.yml
new file mode 100644
index 0000000..ec46d7c
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/tasks/uninstall.yml
@@ -0,0 +1,30 @@
+- name: Check {{ deploy_dir }}/druid/topology
+  stat:
+    path: "{{ deploy_dir }}/druid/topology"
+  register: directory_stat
+
+- block: 
+    - name: Copying supervisor-manager.sh template
+      template:
+        src: 'supervisor-manager.sh.j2'
+        dest: '{{ deploy_dir }}/druid/topology/supervisor-manager.sh'
+        mode: 0755
+
+    - name: Stop all tasks
+      shell: cd {{ deploy_dir }}/druid/topology && sh supervisor-manager.sh terminateall
+      run_once: true
+    
+    - name: Get tasks status
+      shell: curl -s http://{{ vrrp_instance.default.virtual_ipaddress }}:8089/druid/indexer/v1/supervisor?state=true | jq 'map(select(.state = "RUNNING" ))' | jq 'length'    
+      register: tasks_count
+
+    - name: Check tasks status
+      debug: 
+        msg: "{{ tasks_count.stdout }} tasks are not stopped"
+      failed_when: tasks_count.stdout > '0'
+
+    - name: Delete {{ deploy_dir }}/druid/topology
+      file:
+         path: "{{ deploy_dir }}/druid/topology"
+         state: absent
+  when: directory_stat.stat.exists
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/druid_monitor.sh.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/druid_monitor.sh.j2
new file mode 100644
index 0000000..e0c0444
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/druid_monitor.sh.j2
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+druid_ip={{ vrrp_instance.default.virtual_ipaddress }}
+druid_port=8089
+druid_monitor_prom_file={{ deploy_dir }}/node-exporter/prom/druid_monitor.prom
+
+task_name=`cat tasklist`
+
+#获取supervisor状态信息
+function getSupervisorStatus(){
+druid_supervisor_healthy_num=`curl -G -d 'state=true' -s $druid_ip:$druid_port/druid/indexer/v1/supervisor | jq '[ .[] | .state] | length'`
+echo druid_supervisor_healthy_num $druid_supervisor_healthy_num >> $druid_monitor_prom_file
+
+druid_supervisor_unhealthy_num=`curl -G -d 'state=true' -s $druid_ip:$druid_port/druid/indexer/v1/supervisor | jq '[ .[] | select(.state != "RUNNING")] | length'`
+echo druid_supervisor_unhealthy_num $druid_supervisor_unhealthy_num >> $druid_monitor_prom_file
+}
+
+#获取Druid 任务状态信息
+function getTaskStatus(){
+druid_index_running_task_num=`curl -G -d 'type=index_kafka' -s $druid_ip:$druid_port/druid/indexer/v1/runningTasks | jq '. | length'`
+echo druid_index_running_task_num $druid_index_running_task_num > $druid_monitor_prom_file
+
+druid_index_waiting_task_num=`curl -G -d 'type=index_kafka' -s $druid_ip:$druid_port/druid/indexer/v1/waitingTasks | jq '. | length'`
+echo druid_index_waiting_task_num $druid_index_waiting_task_num >> $druid_monitor_prom_file
+
+druid_index_pending_task_num=`curl -G -d 'type=index_kafka' -s $druid_ip:$druid_port/druid/indexer/v1/pendingTasks | jq '. | length'`
+echo druid_index_pending_task_num $druid_index_pending_task_num >> $druid_monitor_prom_file
+
+druid_compact_pending_task_num=`curl -G -d 'type=compact' -s $druid_ip:$druid_port/druid/indexer/v1/pendingTasks | jq '. | length'`
+echo druid_compact_pending_task_num $druid_compact_pending_task_num >> $druid_monitor_prom_file
+
+druid_compact_waiting_task_num=`curl -G -d 'type=compact' -s $druid_ip:$druid_port/druid/indexer/v1/waitingTasks | jq '. | length'`
+echo druid_compact_waiting_task_num $druid_compact_waiting_task_num >> $druid_monitor_prom_file
+}
+
+#校验正在运行的index数量，与启动时指定的index数量对比。
+#>=指定数值判断为正常 返回：0
+#反之异常 返回：1
+function checkIndexStatus(){
+druid_task_sum=0
+for var in ${task_name[@]};
+do
+	druid_task_num=`cat tasks/${var} | jq .ioConfig.taskCount`
+	druid_task_sum=`expr $druid_task_num + $druid_task_sum`
+done
+
+druid_index_task_healthy_flag=0
+if [ $druid_index_running_task_num -lt $druid_task_sum ]
+then
+	druid_index_task_healthy_flag=1
+fi
+
+echo druid_index_task_healthy_flag $druid_index_task_healthy_flag >> $druid_monitor_prom_file
+}
+
+getTaskStatus
+getSupervisorStatus
+checkIndexStatus
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/rule/post_rule.sh.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/rule/post_rule.sh.j2
new file mode 100644
index 0000000..bbffdc4
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/rule/post_rule.sh.j2
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+base_dir=`dirname $0`
+echo $base_dir
+
+#router节点IP
+router_ip={{ groups.druid[0] }}
+
+common_task_name=`cat $base_dir/common_task.txt`
+common_rule_file="$base_dir/common_data_rule.json"
+
+hot_task_name=`cat $base_dir/hot_task.txt`
+hot_rule_file="$base_dir/hot_data_rule.json"
+
+for var in ${common_task_name[@]};
+do
+curl --retry-delay 3 --retry 3 -X 'POST' -H 'Content-Type:application/json' -d @$common_rule_file http://${router_ip}:8088/druid/coordinator/v1/rules/${var}
+echo "'${var}' 任务启动成功" 
+sleep 2
+done
+
+for var in ${hot_task_name[@]};
+do
+curl --retry-delay 3 --retry 3 -X 'POST' -H 'Content-Type:application/json' -d @$hot_rule_file http://${router_ip}:8088/druid/coordinator/v1/rules/${var}
+echo "'${var}' 任务启动成功" 
+sleep 2
+done
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/rule/supervisor-manager.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/rule/supervisor-manager.j2
new file mode 100644
index 0000000..88da982
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/rule/supervisor-manager.j2
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+base_dir=`dirname $0`
+base_dir="$(cd "$base_dir" && pwd)"
+
+router_ip='{{ vrrp_instance.default.virtual_ipaddress }}'
+
+common_task_name=`cat $base_dir/tasklist`
+common_rule_file="$base_dir/common_data_rule.json"
+
+hot_task_name=`cat $base_dir/hot_task.txt`
+hot_rule_file="$base_dir/hot_data_rule.json"
+
+case $1 in
+
+	resetAll)
+	for var in ${common_task_name[@]};
+	do
+		curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/${var}/reset
+		sleep 2
+	done
+
+	for var in ${hot_task_name[@]};
+	do
+		curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/${var}/reset
+		sleep 2
+	done
+	;;
+
+	terminateAll)
+		curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/terminateAll
+	;;
+	
+	shutdownAllTasks)
+	for var in ${common_task_name[@]};
+        do
+                curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/${var}/shutdownAllTasks
+                sleep 2
+        done
+
+        for var in ${hot_task_name[@]};
+        do
+                curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/${var}/shutdownAllTasks
+                sleep 2
+        done
+	;;
+	
+	reset)
+		if [ $# -le 1 ]; then
+  			echo "Usage: supervisor-manger reset <dataSource>"
+  			exit 1
+		fi
+		curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/$2/reset
+	;;
+	
+	terminate)
+		if [ $# -le 1 ]; then
+                        echo "Usage: supervisor-manger terminate <dataSource>"
+                        exit 1
+                fi
+		curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/supervisor/$2/terminate
+	;;
+
+	shutdownTasks)
+                if [ $# -le 1 ]; then
+                        echo "Usage: supervisor-manger shutdownTasks <dataSource>"
+                        exit 1
+                fi
+		curl --retry-delay 3 --retry 3 -X 'POST' http://${router_ip}:8089/druid/indexer/v1/datasources/$2/shutdownAllTasks
+	;;
+	*)
+		echo "Usage: supervisor-manger {resetAll|terminateAll|shutdownAllTasks}"
+                echo "Usage: supervisor-manger {reset|terminate|shutdownTasks} <dataSource>"
+	;;
+esac
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/run_druid_monitor.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/run_druid_monitor.j2
new file mode 100644
index 0000000..d5fee98
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/run_druid_monitor.j2
@@ -0,0 +1,3 @@
+#Ansible: Check druid up to node_exporter
+*/10 * * * * root cd {{ topology_path }} && sh druid_monitor.sh
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/s.bak b/tsg-olap/parcels/roles/initialization/druid/templates/s.bak
new file mode 100644
index 0000000..ed547af
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/s.bak
@@ -0,0 +1,124 @@
+#!/bin/bash
+source /etc/profile
+
+DRUID_HOST='{{ vrrp_instance.default.virtual_ipaddress }}'
+KAFKA_HOST="{% for host in groups['kafka'] %}
+{{ host }}:9094{% if not loop.last %},{% endif %}
+{% endfor %}"
+
+# 参数：1.操作 2.任务文件
+OPERATION=$1
+
+TASK_LIST=(
+    index_kafka_application_protocol_stat.json
+    index_kafka_monitor_rule_hits.json
+    index_kafka_object_statistics.json
+    index_kafka_proxy_rule_hits.json
+    index_kafka_security_rule_hits.json
+    index_kafka_service_chaining_rule_hits.json
+    index_kafka_service_function_status.json
+    index_kafka_statistics_rule_hits.json
+    index_kafka_statistics_rule.json
+    index_kafka_sys_storage_log.json
+    index_kafka_top_client_ips.json
+    index_kafka_top_external_ips.json
+    index_kafka_top_internal_ips.json
+    index_kafka_top_server_domains.json
+    index_kafka_top_server_fqdns.json
+    index_kafka_top_server_ips.json
+    index_kafka_top_subscribers.json
+    index_kafka_traffic_general_stat.json
+    index_kafka_traffic_shaping_rule_hits.json
+    index_kafka_traffic_top_destination_ip_metrics_log.json
+)
+
+if [ $# -lt 1 ]; then
+    echo "Usage: ./supervisor-manager.sh [startall|resetall|terminateall|shutdownall]"
+    echo "Usage: ./supervisor-manager.sh [start|reset|terminate|shutdown] <dataSource>"
+    exit 1
+fi
+
+function start_task() {
+    local task_name=$1
+    cp ./tasks/${task_name} ./
+
+    sed -i 's/kafkabootstrap/'$KAFKA_HOST'/' ${task_name}
+
+    curl --retry-delay 3 --retry 3 -X 'POST' -H 'Content-Type:application/json' -d @${task_name} http://$DRUID_HOST:8089/druid/indexer/v1/supervisor
+
+    if [ $? -eq "0" ]; then
+        echo "task:'${task_name}' Start-up success! "
+        rm -rf ./${task_name}
+    else
+        echo "task:'${task_name}' Start-up failure! "
+        exit 1
+    fi
+
+}
+
+function reset_task() {
+    local task_name=$1
+    task_name="${task_name/index_kafka_/}"
+    task_name="${task_name/.json/}"
+    curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/${task_name}/reset"
+    sleep 2
+}
+
+function terminate_task() {
+    local task_name=$1
+    task_name="${task_name/index_kafka_/}"
+    task_name="${task_name/.json/}"
+    curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/${task_name}/terminate"
+}
+
+function terminate_all() {
+    curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/terminateAll"
+}
+
+function shutdown_tasks() {
+    local task_name=$1
+    task_name="${task_name/index_kafka_/}"
+    task_name="${task_name/.json/}"
+    local task_name=$1
+    curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/datasources/${task_name}/shutdownAllTasks"
+    sleep 2
+}
+
+case $OPERATION in
+start)
+    start_task "$2"
+    ;;
+startall)
+    for task in ${TASK_LIST[@]}; do
+        start_task "$task"
+    done
+    ;;
+reset)
+    reset_task "$2"
+    ;;
+resetall)
+    for task in ${TASK_LIST[@]}; do
+        reset_task "$task"
+    done
+    ;;
+terminate)
+    terminate_task "$2"
+    ;;
+terminateall)
+    terminate_all
+    ;;
+shutdown)
+    shutdown_tasks "$2"
+    ;;
+shutdownall)
+    for task in ${TASK_LIST[@]}; do
+        shutdown_tasks "$task"
+    done
+    ;;
+
+*)
+    echo "Usage: ./supervisor-manager.sh [startall|resetall|terminateall|shutdownall]"
+    echo "Usage: ./supervisor-manager.sh [start|reset|terminate|shutdown] <dataSource>"
+    ;;
+esac
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/supervisor-manager.sh.j2 b/tsg-olap/parcels/roles/initialization/druid/templates/supervisor-manager.sh.j2
new file mode 100644
index 0000000..e140a07
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/supervisor-manager.sh.j2
@@ -0,0 +1,103 @@
+#!/bin/bash
+source /etc/profile
+
+DRUID_HOST='{{ vrrp_instance.default.virtual_ipaddress }}'
+KAFKA_HOST="{% for host in groups['kafka'] %}
+{{ host }}:9094{% if not loop.last %},{% endif %}
+{% endfor %}"
+
+# 参数：1.操作 2.任务文件或目录
+OPERATION=$1
+filename=$(basename "$file")
+TASK_FILE_LIST=$(find "./tasks" -name "*.json")
+
+if [ $# -lt 1 ]; then
+    echo "Usage: ./supervisor-manager.sh [startall|resetall|terminateall|shutdownall]"
+    echo "Usage: ./supervisor-manager.sh [start|reset|terminate|shutdown] <./tasks/[task file]>"
+    exit 1
+fi
+
+function start_task() {
+    local task_file=$1
+    local task_name=$(basename $task_file)
+    cp ${task_file} ./
+
+    sed -i 's/kafkabootstrap/'$KAFKA_HOST'/' ${task_name}
+
+    curl --retry-delay 3 --retry 3 -X 'POST' -H 'Content-Type:application/json' -d @${task_name} http://$DRUID_HOST:8089/druid/indexer/v1/supervisor
+
+    if [ $? -eq "0" ]; then
+        echo "task:'${task_name}' Start-up success! "
+        rm -rf ./${task_name}
+    else
+        echo "task:'${task_name}' Start-up failure! "
+        exit 1
+    fi
+
+}
+
+function reset_task() {
+    local task_name=$(basename $1)
+    task_name="${task_name/index_kafka_/}"
+    task_name="${task_name/.json/}"
+    curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/${task_name}/reset"
+    sleep 2
+}
+
+function terminate_task() {
+    local task_name=$(basename $1)
+    task_name="${task_name/index_kafka_/}"
+    task_name="${task_name/.json/}"
+    curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/${task_name}/terminate"
+}
+
+function terminate_all() {
+    curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/supervisor/terminateAll"
+}
+
+function shutdown_tasks() {
+    local task_name=$(basename $1)
+    task_name="${task_name/index_kafka_/}"
+    task_name="${task_name/.json/}"
+    curl --retry-delay 3 --retry 3 -X 'POST' "http://${DRUID_HOST}:8089/druid/indexer/v1/datasources/${task_name}/shutdownAllTasks"
+    sleep 2
+}
+
+case $OPERATION in
+start)
+    start_task "$2"
+    ;;
+startall)
+    for task in ${TASK_FILE_LIST[@]}; do
+        start_task "$task"
+    done
+    ;;
+reset)
+    reset_task "$2"
+    ;;
+resetall)
+    for task in ${TASK_FILE_LIST[@]}; do
+        reset_task "$task"
+    done
+    ;;
+terminate)
+    terminate_task "$2"
+    ;;
+terminateall)
+    terminate_all
+    ;;
+shutdown)
+    shutdown_tasks "$2"
+    ;;
+shutdownall)
+    for task in ${TASK_FILE_LIST[@]}; do
+        shutdown_tasks "$task"
+    done
+    ;;
+
+*)
+    echo "Usage: ./supervisor-manager.sh [startall|resetall|terminateall|shutdownall]"
+    echo "Usage: ./supervisor-manager.sh [start|reset|terminate|shutdown] <./tasks/[task file]>"
+    ;;
+esac
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_application_protocol_stat.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_application_protocol_stat.json
new file mode 100644
index 0000000..96f992a
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_application_protocol_stat.json
@@ -0,0 +1,131 @@
+{
+    "type": "kafka",
+    "dataSchema": {
+        "dataSource": "application_protocol_stat",
+        "parser": {
+            "type": "string",
+            "parseSpec": {
+                "format": "json",
+                "timestampSpec": {
+                    "column": "timestamp_ms",
+                    "format": "millis"
+                },
+                "dimensionsSpec": {
+                    "dimensions": [
+                        "device_id",
+                        "device_group",
+                        "data_center",
+                        "protocol_stack_id",
+                        "app_name",
+                        {"name": "vsys_id", "type": "long"}
+                    ]
+                },
+                "flattenSpec": {
+                    "useFieldDiscovery": true,
+                    "fields": [
+                        {"name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id"},
+                        {"name": "data_center", "type": "path", "expr": "$.tags.data_center"},
+                        {"name": "device_group", "type": "path", "expr": "$.tags.device_group"},
+                        {"name": "device_id", "type": "path", "expr": "$.tags.device_id"},
+                        {"name": "protocol_stack_id", "type": "path", "expr": "$.tags.protocol_stack_id"},
+                        {"name": "app_name", "type": "path", "expr": "$.tags.app_name"},
+                        {"name": "sessions", "type": "path", "expr": "$.fields.sessions"},
+                        {"name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes"},
+                        {"name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes"},
+                        {"name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts"},
+                        {"name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts"},
+                        {"name": "c2s_pkts", "type": "path", "expr": "$.fields.c2s_pkts"},
+                        {"name": "s2c_pkts", "type": "path", "expr": "$.fields.s2c_pkts"},
+                        {"name": "c2s_bytes", "type": "path", "expr": "$.fields.c2s_bytes"},
+                        {"name": "s2c_bytes", "type": "path", "expr": "$.fields.s2c_bytes"},
+                        {"name": "c2s_fragments", "type": "path", "expr": "$.fields.c2s_fragments"},
+                        {"name": "s2c_fragments", "type": "path", "expr": "$.fields.s2c_fragments"},
+                        {"name": "c2s_tcp_lost_bytes", "type": "path", "expr": "$.fields.c2s_tcp_lost_bytes"},
+                        {"name": "s2c_tcp_lost_bytes", "type": "path", "expr": "$.fields.s2c_tcp_lost_bytes"},
+                        {"name": "c2s_tcp_ooorder_pkts", "type": "path", "expr": "$.fields.c2s_tcp_ooorder_pkts"},
+                        {"name": "s2c_tcp_ooorder_pkts", "type": "path", "expr": "$.fields.s2c_tcp_ooorder_pkts"},
+                        {"name": "c2s_tcp_retransmitted_pkts", "type": "path", "expr": "$.fields.c2s_tcp_retransmitted_pkts"},
+                        {"name": "s2c_tcp_retransmitted_pkts", "type": "path", "expr": "$.fields.s2c_tcp_retransmitted_pkts"},
+                        {"name": "c2s_tcp_retransmitted_bytes", "type": "path", "expr": "$.fields.c2s_tcp_retransmitted_bytes"},
+                        {"name": "s2c_tcp_retransmitted_bytes", "type": "path", "expr": "$.fields.s2c_tcp_retransmitted_bytes"}
+                    ]
+                }
+            }
+        },
+        "metricsSpec": [
+            {"type": "longSum", "name": "sessions", "fieldName": "sessions"},
+            {"type": "longSum", "name": "c2s_pkts", "fieldName": "c2s_pkts"},
+            {"type": "longSum", "name": "s2c_pkts", "fieldName": "s2c_pkts"},
+            {"type": "longSum", "name": "c2s_bytes", "fieldName": "c2s_bytes"},
+            {"type": "longSum", "name": "s2c_bytes", "fieldName": "s2c_bytes"},
+            {"type": "longSum", "name": "c2s_fragments", "fieldName": "c2s_fragments"},
+            {"type": "longSum", "name": "s2c_fragments", "fieldName": "s2c_fragments"},
+            {"type": "longSum", "name": "c2s_tcp_lost_bytes", "fieldName": "c2s_tcp_lost_bytes"},
+            {"type": "longSum", "name": "s2c_tcp_lost_bytes", "fieldName": "s2c_tcp_lost_bytes"},
+            {"type": "longSum", "name": "c2s_tcp_ooorder_pkts", "fieldName": "c2s_tcp_ooorder_pkts"},
+            {"type": "longSum", "name": "s2c_tcp_ooorder_pkts", "fieldName": "s2c_tcp_ooorder_pkts"},
+            {"type": "longSum", "name": "c2s_tcp_retransmitted_pkts", "fieldName": "c2s_tcp_retransmitted_pkts"},
+            {"type": "longSum", "name": "s2c_tcp_retransmitted_pkts", "fieldName": "s2c_tcp_retransmitted_pkts"},
+            {"type": "longSum", "name": "c2s_tcp_retransmitted_bytes", "fieldName": "c2s_tcp_retransmitted_bytes"},
+            {"type": "longSum", "name": "s2c_tcp_retransmitted_bytes", "fieldName": "s2c_tcp_retransmitted_bytes"},
+            {"type": "longSum", "name": "in_bytes", "fieldName": "in_bytes"},
+            {"type": "longSum", "name": "out_bytes", "fieldName": "out_bytes"},
+            {"type": "longSum", "name": "in_pkts", "fieldName": "in_pkts"},
+            {"type": "longSum", "name": "out_pkts", "fieldName": "out_pkts"}
+        ],
+        "granularitySpec": {
+            "type": "uniform",
+            "segmentGranularity": "DAY",
+            "queryGranularity": {"type": "period", "period": "PT15S"},
+            "rollup": true
+        },
+        "transformSpec": {
+            "transforms": [
+                {"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"},
+                {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
+                {"type": "expression", "name": "c2s_pkts", "expression": "nvl(c2s_pkts, 0)"},
+                {"type": "expression", "name": "s2c_pkts", "expression": "nvl(s2c_pkts, 0)"},
+                {"type": "expression", "name": "c2s_bytes", "expression": "nvl(c2s_bytes, 0)"},
+                {"type": "expression", "name": "s2c_bytes", "expression": "nvl(s2c_bytes, 0)"},
+                {"type": "expression", "name": "c2s_fragments", "expression": "nvl(c2s_fragments, 0)"},
+                {"type": "expression", "name": "s2c_fragments", "expression": "nvl(s2c_fragments, 0)"},
+                {"type": "expression", "name": "c2s_tcp_lost_bytes", "expression": "nvl(c2s_tcp_lost_bytes, 0)"},
+                {"type": "expression", "name": "s2c_tcp_lost_bytes", "expression": "nvl(s2c_tcp_lost_bytes, 0)"},
+                {"type": "expression", "name": "c2s_tcp_ooorder_pkts", "expression": "nvl(c2s_tcp_ooorder_pkts, 0)"},
+                {"type": "expression", "name": "s2c_tcp_ooorder_pkts", "expression": "nvl(s2c_tcp_ooorder_pkts, 0)"},
+                {"type": "expression", "name": "c2s_tcp_retransmitted_pkts", "expression": "nvl(c2s_tcp_retransmitted_pkts, 0)"},
+                {"type": "expression", "name": "s2c_tcp_retransmitted_pkts", "expression": "nvl(s2c_tcp_retransmitted_pkts, 0)"},
+                {"type": "expression", "name": "c2s_tcp_retransmitted_bytes", "expression": "nvl(c2s_tcp_retransmitted_bytes, 0)"},
+                {"type": "expression", "name": "s2c_tcp_retransmitted_bytes", "expression": "nvl(s2c_tcp_retransmitted_bytes, 0)"},
+                {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
+                {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
+                {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
+                {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
+            ],
+            "filter": {
+                "type": "selector",
+                "dimension": "name",
+                "value": "application_protocol_stat"
+            }
+        }
+    },
+    "tuningConfig": {
+        "type": "kafka",
+        "maxRowsPerSegment": 5000000,
+        "resetOffsetAutomatically": true,
+        "reportParseExceptions": false
+    },
+    "ioConfig": {
+        "topic": "NETWORK-TRAFFIC-METRIC",
+        "taskCount": 1,
+        "replicas": 1,
+        "taskDuration": "PT1H",
+        "completionTimeout": "PT30M",
+        "consumerProperties": {
+            "bootstrap.servers": "kafkabootstrap",
+            "sasl.mechanism": "PLAIN",
+            "security.protocol": "SASL_PLAINTEXT",
+            "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+        }
+    }
+}
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_monitor_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_monitor_rule_hits.json
new file mode 100644
index 0000000..5c43f46
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_monitor_rule_hits.json
@@ -0,0 +1,83 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "monitor_rule_hits",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "timestamp_ms",
+          "format": "millis"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {"name": "vsys_id","type": "long"},
+            "device_id",
+            "device_group",
+            "data_center",
+            {"name": "rule_id","type": "long"},
+            {"name": "action","type": "long"}
+          ]
+        },
+    "flattenSpec": {
+        "useFieldDiscovery": true,
+        "fields": [
+                { "name": "hit_count", "type": "path", "expr": "$.fields.hit_count" },
+                { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" },
+                { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" },
+                { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" },
+                { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" },
+                { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" },
+                { "name": "data_center", "type": "path", "expr": "$.tags.data_center" },
+                { "name": "device_group", "type": "path", "expr": "$.tags.device_group" },
+                { "name": "device_id", "type": "path", "expr": "$.tags.device_id" },
+                { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" },
+                { "name": "action", "type": "path", "expr": "$.tags.action" }
+        ]
+    }
+      }
+    },
+    "metricsSpec" : [
+        { "type" : "longSum", "name" : "hit_count", "fieldName" : "hit_count" },
+        { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
+        { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
+        { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
+        { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
+  ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT1S"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[
+        {"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"},
+        {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
+        {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
+        {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
+        {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
+      ],
+      "filter": { "type": "selector", "dimension": "name", "value": "monitor_rule_hits" }
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "POLICY-RULE-METRIC",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap",
+      "sasl.mechanism": "PLAIN",
+      "security.protocol": "SASL_PLAINTEXT",
+      "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+    }
+  }
+}
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_object_statistics.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_object_statistics.json
new file mode 100644
index 0000000..84dd11b
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_object_statistics.json
@@ -0,0 +1,95 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "object_statistics",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "timestamp_ms",
+          "format": "millis"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {"name": "vsys_id","type": "long"},
+            "device_id",
+            "device_group",
+            "data_center",
+            {"name": "template_id","type": "long"},
+            {"name": "chart_id","type": "long"},
+            {"name": "version","type": "long"},
+            "object_type",
+            {"name": "object_id","type": "long"},
+            {"name": "item_id","type": "long"}
+          ]
+        },
+        "flattenSpec": {
+            "useFieldDiscovery": true,
+            "fields": [
+                { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" },
+                { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" },
+                { "name": "bytes", "type": "path", "expr": "$.fields.bytes" },
+                { "name": "new_in_sessions", "type": "path", "expr": "$.fields.new_in_sessions" },
+                { "name": "new_out_sessions", "type": "path", "expr": "$.fields.new_out_sessions" },
+                { "name": "sessions", "type": "path", "expr": "$.fields.sessions" },
+                { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" },
+                { "name": "data_center", "type": "path", "expr": "$.tags.data_center" },
+                { "name": "device_group", "type": "path", "expr": "$.tags.device_group" },
+                { "name": "device_id", "type": "path", "expr": "$.tags.device_id" },
+                { "name": "template_id", "type": "path", "expr": "$.tags.template_id" },
+                { "name": "chart_id", "type": "path", "expr": "$.tags.chart_id" },
+                { "name": "version", "type": "path", "expr": "$.tags.version" },
+                { "name": "object_type", "type": "path", "expr": "$.tags.object_type" },
+                { "name": "object_id", "type": "path", "expr": "$.tags.object_id" },
+                { "name": "item_id", "type": "path", "expr": "$.tags.item_id" }
+            ]
+        }
+      }
+    },
+    "metricsSpec" : [
+        { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
+        { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
+        { "type" : "longSum", "name" : "bytes", "fieldName" : "bytes" },
+        { "type" : "longSum", "name" : "new_in_sessions", "fieldName" : "new_in_sessions" },
+        { "type" : "longSum", "name" : "new_out_sessions", "fieldName" : "new_out_sessions" },
+        { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" }
+  ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT15S"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[
+        {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
+        {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
+        {"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"},
+        {"type": "expression", "name": "new_in_sessions", "expression": "nvl(new_in_sessions, 0)"},
+        {"type": "expression", "name": "new_out_sessions", "expression": "nvl(new_out_sessions, 0)"},
+        {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"}
+      ],
+      "filter": { "type": "selector", "dimension": "name", "value": "object_statistics" }
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "OBJECT-STATISTICS-METRIC",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap",
+      "sasl.mechanism": "PLAIN",
+      "security.protocol": "SASL_PLAINTEXT",
+      "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+    }
+  }
+}
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_proxy_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_proxy_rule_hits.json
new file mode 100644
index 0000000..ab97fb1
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_proxy_rule_hits.json
@@ -0,0 +1,90 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "proxy_rule_hits",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "timestamp_ms",
+          "format": "millis"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+           "device_id",
+           "device_group",
+           "data_center",
+           { "name": "vsys_id", "type": "long" },
+           { "name": "rule_id", "type": "long" },
+           { "name": "pinning_status", "type": "string" },
+           { "name": "action", "type": "long" },
+           { "name": "sub_action", "type": "string" }
+          ]
+        },
+          "flattenSpec": {
+  	  "useFieldDiscovery": true,
+  	  "fields": [
+                { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" },
+                { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" },
+                { "name": "data_center", "type": "path", "expr": "$.tags.data_center" },
+                { "name": "device_group", "type": "path", "expr": "$.tags.device_group" },
+                { "name": "device_id", "type": "path", "expr": "$.tags.device_id" },
+                { "name": "pinning_status", "type": "path", "expr": "$.tags.pinning_status" },
+		{ "name": "action", "type": "path", "expr": "$.tags.action" },
+                { "name": "sub_action", "type": "path", "expr": "$.tags.sub_action" },
+		{ "name": "hit_count", "type": "path", "expr": "$.fields.hit_count" },
+                { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" },
+                { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" },
+                { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" },
+                { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" }
+  	  ]
+	}
+      }
+    },
+    "metricsSpec" : [
+	{ "type" : "longSum", "name" : "hit_count", "fieldName" : "hit_count" },
+	{ "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
+	{ "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
+        { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
+        { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
+  ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT1S"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[
+        {"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"},
+        {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
+        {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
+        {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
+        {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
+      ],
+      "filter": { "type": "selector", "dimension": "name", "value": "proxy_rule_hits" }
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "maxRowsPerSegment": 5000000,
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "POLICY-RULE-METRIC",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap",
+      "sasl.mechanism": "PLAIN",
+      "security.protocol": "SASL_PLAINTEXT",
+      "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+    }
+  }
+}
+
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_security_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_security_rule_hits.json
new file mode 100644
index 0000000..957923d
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_security_rule_hits.json
@@ -0,0 +1,83 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "security_rule_hits",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "timestamp_ms",
+          "format": "millis"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {"name": "vsys_id","type": "long"},
+            "device_id",
+            "device_group",
+            "data_center",
+            {"name": "rule_id","type": "long"},
+            {"name": "action","type": "long"}
+          ]
+        },
+    "flattenSpec": {
+        "useFieldDiscovery": true,
+        "fields": [
+                { "name": "hit_count", "type": "path", "expr": "$.fields.hit_count" },
+                { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" },
+                { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" },
+                { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" },
+                { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" },
+                { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" },
+                { "name": "data_center", "type": "path", "expr": "$.tags.data_center" },
+                { "name": "device_group", "type": "path", "expr": "$.tags.device_group" },
+                { "name": "device_id", "type": "path", "expr": "$.tags.device_id" },
+                { "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" },
+                { "name": "action", "type": "path", "expr": "$.tags.action" }
+        ]
+    }
+      }
+    },
+    "metricsSpec" : [
+        { "type" : "longSum", "name" : "hit_count", "fieldName" : "hit_count" },
+        { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
+        { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
+        { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
+        { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
+  ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT1S"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[
+        {"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"},
+        {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
+        {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
+        {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
+        {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
+      ],
+      "filter": { "type": "selector", "dimension": "name", "value": "security_rule_hits" }
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "POLICY-RULE-METRIC",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap",
+      "sasl.mechanism": "PLAIN",
+      "security.protocol": "SASL_PLAINTEXT",
+      "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+    }
+  }
+}
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_chaining_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_chaining_rule_hits.json
new file mode 100644
index 0000000..6f3e3bd
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_chaining_rule_hits.json
@@ -0,0 +1,84 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "service_chaining_rule_hits",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "timestamp_ms",
+          "format": "millis"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+			"device_id",
+			"device_group",
+			"data_center",
+           { "name": "vsys_id", "type": "long" },
+           { "name": "rule_id", "type": "long" },
+           { "name": "sff_profile_id", "type": "long" },
+           { "name": "sf_profile_id", "type": "long" }
+          ]
+        },
+		"flattenSpec": {
+  	  "useFieldDiscovery": true,
+  	  "fields": [
+                { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" },
+                { "name": "data_center", "type": "path", "expr": "$.tags.data_center" },
+                { "name": "device_group", "type": "path", "expr": "$.tags.device_group" },
+                { "name": "device_id", "type": "path", "expr": "$.tags.device_id" },
+				{ "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" },
+				{ "name": "sff_profile_id", "type": "path", "expr": "$.tags.sff_profile_id" },
+				{ "name": "sf_profile_id", "type": "path", "expr": "$.tags.sf_profile_id" },
+                { "name": "sent_pkts", "type": "path", "expr": "$.fields.sent_pkts" },
+                { "name": "sent_bytes", "type": "path", "expr": "$.fields.sent_bytes" },
+                { "name": "recv_pkts", "type": "path", "expr": "$.fields.recv_pkts" },
+                { "name": "recv_bytes", "type": "path", "expr": "$.fields.recv_bytes" }
+  	  ]
+	}
+      }
+    },
+    "metricsSpec" : [
+	{ "type" : "longSum", "name" : "sent_pkts", "fieldName" : "sent_pkts" },
+	{ "type" : "longSum", "name" : "sent_bytes", "fieldName" : "sent_bytes" },
+        { "type" : "longSum", "name" : "recv_pkts", "fieldName" : "recv_pkts" },
+        { "type" : "longSum", "name" : "recv_bytes", "fieldName" : "recv_bytes" }
+  ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT1S"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[
+        {"type": "expression", "name": "sent_pkts", "expression": "nvl(sent_pkts, 0)"},
+        {"type": "expression", "name": "sent_bytes", "expression": "nvl(sent_bytes, 0)"},
+        {"type": "expression", "name": "recv_pkts", "expression": "nvl(recv_pkts, 0)"},
+        {"type": "expression", "name": "recv_bytes", "expression": "nvl(recv_bytes, 0)"}
+      ],
+      "filter": { "type": "selector", "dimension": "name", "value": "service_chaining_rule_hits" }
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "maxRowsPerSegment": 5000000,
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "POLICY-RULE-METRIC",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap",
+      "sasl.mechanism": "PLAIN",
+      "security.protocol": "SASL_PLAINTEXT",
+      "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+    }
+  }
+}
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_function_status.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_function_status.json
new file mode 100644
index 0000000..61e5b12
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_service_function_status.json
@@ -0,0 +1,74 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "service_function_status",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "timestamp_ms",
+          "format": "millis"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+	   "device_id",
+	   "device_group",
+	   "data_center",
+           { "name": "vsys_id", "type": "long" },
+           { "name": "sf_profile_id", "type": "long" }
+          ]
+        },
+		"flattenSpec": {
+  	  "useFieldDiscovery": true,
+  	  "fields": [
+                { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" },
+                { "name": "data_center", "type": "path", "expr": "$.tags.data_center" },
+                { "name": "device_group", "type": "path", "expr": "$.tags.device_group" },
+                { "name": "device_id", "type": "path", "expr": "$.tags.device_id" },
+				{ "name": "sf_profile_id", "type": "path", "expr": "$.tags.sf_profile_id" },
+                { "name": "sf_status", "type": "path", "expr": "$.fields.sf_status" },
+                { "name": "sf_latency_us", "type": "path", "expr": "$.fields.sf_latency_us" }
+  	  ]
+	}
+      }
+    },
+    "metricsSpec" : [
+	{ "type" : "longMax", "name" : "sf_latency_us", "fieldName" : "sf_latency_us" },
+	{ "type" : "longMax", "name" : "sf_status", "fieldName" : "sf_status" }
+  ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT1S"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[
+        {"type": "expression", "name": "sf_latency_us", "expression": "nvl(sf_latency_us, 0)"},
+        {"type": "expression", "name": "sf_status", "expression": "nvl(sf_status, 0)"}
+      ],
+      "filter": { "type": "selector", "dimension": "name", "value": "service_function_status" }
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "maxRowsPerSegment": 5000000,
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "POLICY-RULE-METRIC",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap",
+      "sasl.mechanism": "PLAIN",
+      "security.protocol": "SASL_PLAINTEXT",
+      "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+    }
+  }
+}
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule.json
new file mode 100644
index 0000000..8c15a37
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule.json
@@ -0,0 +1,367 @@
+{
+    "type": "kafka",
+    "dataSchema": {
+        "dataSource": "statistics_rule",
+        "parser": {
+            "type": "string",
+            "parseSpec": {
+                "format": "json",
+                "timestampSpec": {
+                    "column": "timestamp_ms",
+                    "format": "millis"
+                },
+                "dimensionsSpec": {
+                    "dimensions": [
+                        "device_id",
+                        "device_group",
+                        "data_center",
+                        {
+                            "name": "vsys_id",
+                            "type": "long"
+                        },
+						{
+                            "name": "rule_id",
+                            "type": "long"
+                        },
+						{
+                            "name": "template_id",
+                            "type": "long"
+                        },
+						{
+                            "name": "chart_id",
+                            "type": "long"
+                        },
+						{
+                            "name": "version",
+                            "type": "long"
+                        },
+						"client_ip_object",
+						"server_ip_object",
+						"fqdn_category",
+						"client_ip",
+						"server_ip",
+						"server_fqdn",
+						"server_domain",
+						"application"
+                    ]
+                },
+                "flattenSpec": {
+                    "useFieldDiscovery": true,
+                    "fields": [
+                        {
+                            "name": "vsys_id",
+                            "type": "path",
+                            "expr": "$.tags.vsys_id"
+                        },
+                        {
+                            "name": "data_center",
+                            "type": "path",
+                            "expr": "$.tags.data_center"
+                        },
+                        {
+                            "name": "device_group",
+                            "type": "path",
+                            "expr": "$.tags.device_group"
+                        },
+                        {
+                            "name": "device_id",
+                            "type": "path",
+                            "expr": "$.tags.device_id"
+                        },
+                        {
+                            "name": "rule_id",
+                            "type": "path",
+                            "expr": "$.tags.rule_id"
+                        },
+                        {
+                            "name": "template_id",
+                            "type": "path",
+                            "expr": "$.tags.template_id"
+                        },
+                        {
+                            "name": "chart_id",
+                            "type": "path",
+                            "expr": "$.tags.chart_id"
+                        },
+                        {
+                            "name": "version",
+                            "type": "path",
+                            "expr": "$.tags.version"
+                        },
+						{
+                            "name": "client_ip_object",
+                            "type": "path",
+                            "expr": "$.tags.client_ip_object"
+                        },
+						{
+                            "name": "server_ip_object",
+                            "type": "path",
+                            "expr": "$.tags.server_ip_object"
+                        },
+						{
+                            "name": "fqdn_category",
+                            "type": "path",
+                            "expr": "$.tags.fqdn_category"
+                        },
+						{
+                            "name": "client_ip",
+                            "type": "path",
+                            "expr": "$.tags.client_ip"
+                        },
+						{
+                            "name": "server_ip",
+                            "type": "path",
+                            "expr": "$.tags.server_ip"
+                        },
+						{
+                            "name": "server_fqdn",
+                            "type": "path",
+                            "expr": "$.tags.server_fqdn"
+                        },
+						{
+                            "name": "server_domain",
+                            "type": "path",
+                            "expr": "$.tags.server_domain"
+                        },
+						{
+                            "name": "application",
+                            "type": "path",
+                            "expr": "$.tags.application"
+                        },
+                        {
+                            "name": "in_bytes",
+                            "type": "path",
+                            "expr": "$.fields.in_bytes"
+                        },
+                        {
+                            "name": "out_bytes",
+                            "type": "path",
+                            "expr": "$.fields.out_bytes"
+                        },
+                        {
+                            "name": "bytes",
+                            "type": "path",
+                            "expr": "$.fields.bytes"
+                        },
+                        {
+                            "name": "new_c2s_flows",
+                            "type": "path",
+                            "expr": "$.fields.new_c2s_flows"
+                        },
+                        {
+                            "name": "new_s2c_flows",
+                            "type": "path",
+                            "expr": "$.fields.new_s2c_flows"
+                        },
+                        {
+                            "name": "new_in_sessions",
+                            "type": "path",
+                            "expr": "$.fields.new_in_sessions"
+                        },
+                        {
+                            "name": "new_out_sessions",
+                            "type": "path",
+                            "expr": "$.fields.new_out_sessions"
+                        },
+                        {
+                            "name": "sessions",
+                            "type": "path",
+                            "expr": "$.fields.sessions"
+                        },
+                        {
+                            "name": "new_unestablished_sessions",
+                            "type": "path",
+                            "expr": "$.fields.new_unestablished_sessions"
+                        },
+                        {
+                            "name": "syn_pkts",
+                            "type": "path",
+                            "expr": "$.fields.syn_pkts"
+                        },
+                        {
+                            "name": "session_identifier_sketch",
+                            "type": "path",
+                            "expr": "$.fields.session_identifier_sketch"
+                        },
+                        {
+                            "name": "client_ip_sketch",
+                            "type": "path",
+                            "expr": "$.fields.client_ip_sketch"
+                        },
+                        {
+                            "name": "server_ip_sketch",
+                            "type": "path",
+                            "expr": "$.fields.server_ip_sketch"
+                        },
+                        {
+                            "name": "in_latency_ms_sketch",
+                            "type": "path",
+                            "expr": "$.fields.in_latency_ms_sketch"
+                        },
+                        {
+                            "name": "out_latency_ms_sketch",
+                            "type": "path",
+                            "expr": "$.fields.out_latency_ms_sketch"
+                        },
+                        {
+                            "name": "latency_ms_sketch",
+                            "type": "path",
+                            "expr": "$.fields.latency_ms_sketch"
+                        }
+                    ]
+                }
+            }
+        },
+        "metricsSpec": [
+            {
+                "type": "HLLDSketchMerge",
+                "name": "session_identifier_sketch",
+                "fieldName": "session_identifier_sketch"
+            },
+			{
+                "type": "HLLDSketchMerge",
+                "name": "client_ip_sketch",
+                "fieldName": "client_ip_sketch"
+            },{
+                "type": "HLLDSketchMerge",
+                "name": "server_ip_sketch",
+                "fieldName": "server_ip_sketch"
+            },
+			{
+                "type": "HdrHistogramSketchMerge",
+                "name": "in_latency_ms_sketch",
+                "fieldName": "in_latency_ms_sketch",
+                 "lowestDiscernibleValue": "1",
+                "highestTrackableValue": "2",
+                "numberOfSignificantValueDigits": "3",
+                 "autoResize": "true"
+            },
+			{
+                "type": "HdrHistogramSketchMerge",
+                "name": "out_latency_ms_sketch",
+                "fieldName": "out_latency_ms_sketch",
+                 "lowestDiscernibleValue": "1",
+                 "highestTrackableValue": "2",
+                "numberOfSignificantValueDigits": "3",
+                "autoResize": "true"
+            },
+			{
+                "type": "HdrHistogramSketchMerge",
+                "name": "latency_ms_sketch",
+                "fieldName": "latency_ms_sketch",
+                 "lowestDiscernibleValue": "1",
+                "highestTrackableValue": "2",
+                "numberOfSignificantValueDigits": "3",
+                 "autoResize": "true"
+            },
+            {
+                "type": "longSum",
+                "name": "in_bytes",
+                "fieldName": "in_bytes"
+            },
+            {
+                "type": "longSum",
+                "name": "out_bytes",
+                "fieldName": "out_bytes"
+            },
+            {
+                "type": "longSum",
+                "name": "bytes",
+                "fieldName": "bytes"
+            },
+            {
+                "type": "longSum",
+                "name": "new_c2s_flows",
+                "fieldName": "new_c2s_flows"
+            },
+            {
+                "type": "longSum",
+                "name": "new_s2c_flows",
+                "fieldName": "new_s2c_flows"
+            },
+            {
+                "type": "longSum",
+                "name": "new_in_sessions",
+                "fieldName": "new_in_sessions"
+            },
+            {
+                "type": "longSum",
+                "name": "new_out_sessions",
+                "fieldName": "new_out_sessions"
+            },
+            {
+                "type": "longSum",
+                "name": "sessions",
+                "fieldName": "sessions"
+            },
+            {
+                "type": "longSum",
+                "name": "new_unestablished_sessions",
+                "fieldName": "new_unestablished_sessions"
+            },
+            {
+                "type": "longSum",
+                "name": "syn_pkts",
+                "fieldName": "syn_pkts"
+            }
+        ],
+        "granularitySpec": {
+            "type": "uniform",
+            "segmentGranularity": "DAY",
+            "queryGranularity": {
+                "type": "period",
+                "period": "PT15S"
+            },
+            "rollup": true
+        },
+        "transformSpec": {
+            "transforms": [
+                {"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"},
+                {"type": "expression", "name": "client_ip_object", "expression": "STRING_TO_ARRAY(client_ip_object,',')"},
+                {"type": "expression", "name": "server_ip_object", "expression": "STRING_TO_ARRAY(server_ip_object,',')"},
+                {"type": "expression", "name": "fqdn_category", "expression": "STRING_TO_ARRAY(fqdn_category,',')"},
+                {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
+                {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
+                {"type": "expression", "name": "bytes", "expression": "nvl(bytes, 0)"},
+                {"type": "expression", "name": "new_c2s_flows", "expression": "nvl(new_c2s_flows, 0)"},
+                {"type": "expression", "name": "new_s2c_flows", "expression": "nvl(new_s2c_flows, 0)"},
+                {"type": "expression", "name": "new_in_sessions", "expression": "nvl(new_in_sessions, 0)"},
+                {"type": "expression", "name": "new_out_sessions", "expression": "nvl(new_out_sessions, 0)"},
+                {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
+                {"type": "expression", "name": "new_unestablished_sessions", "expression": "nvl(new_unestablished_sessions, 0)"},
+                {"type": "expression", "name": "syn_pkts", "expression": "nvl(syn_pkts, 0)"}
+            ],
+            "filter": {
+                "type": "selector",
+                "dimension": "name",
+                "value": "statistics_rule"
+            }
+        }
+    },
+    "tuningConfig": {
+        "type": "kafka",
+        "maxRowsPerSegment": 5000000,
+        "maxRowsInMemory": 400000,
+        "maxBytesInMemory": -1,
+        "resetOffsetAutomatically": true,
+        "reportParseExceptions": false
+    },
+    "ioConfig": {
+        "topic": "STATISTICS-RULE-METRIC",
+        "taskCount": 2,
+        "replicas": 1,
+        "taskDuration": "PT1H",
+        "completionTimeout": "PT30M",
+        "consumerProperties": {
+            "bootstrap.servers": "kafkabootstrap",
+            "sasl.mechanism": "PLAIN",
+            "security.protocol": "SASL_PLAINTEXT",
+            "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+        }
+    },
+    "context":{
+    "druid.indexer.runner.javaOpts": "{{ druid.index_kafka_statistics_rule['druid.indexer.runner.javaOpts'] }}"
+  }
+}
+
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule_hits.json
new file mode 100644
index 0000000..f431e0f
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_statistics_rule_hits.json
@@ -0,0 +1,159 @@
+{
+    "type": "kafka",
+    "dataSchema": {
+        "dataSource": "statistics_rule_hits",
+        "parser": {
+            "type": "string",
+            "parseSpec": {
+                "format": "json",
+                "timestampSpec": {
+                    "column": "timestamp_ms",
+                    "format": "millis"
+                },
+                "dimensionsSpec": {
+                    "dimensions": [
+                        "device_id",
+                        "device_group",
+                        "data_center",
+                        {
+                            "name": "vsys_id",
+                            "type": "long"
+                        },
+						{
+                            "name": "rule_id",
+                            "type": "long"
+                        }
+                    ]
+                },
+                "flattenSpec": {
+                    "useFieldDiscovery": true,
+                    "fields": [
+                        {
+                            "name": "vsys_id",
+                            "type": "path",
+                            "expr": "$.tags.vsys_id"
+                        },
+                        {
+                            "name": "data_center",
+                            "type": "path",
+                            "expr": "$.tags.data_center"
+                        },
+                        {
+                            "name": "device_group",
+                            "type": "path",
+                            "expr": "$.tags.device_group"
+                        },
+                        {
+                            "name": "device_id",
+                            "type": "path",
+                            "expr": "$.tags.device_id"
+                        },
+                        {
+                            "name": "rule_id",
+                            "type": "path",
+                            "expr": "$.tags.rule_id"
+                        },
+                        {
+                            "name": "in_bytes",
+                            "type": "path",
+                            "expr": "$.fields.in_bytes"
+                        },
+                        {
+                            "name": "out_bytes",
+                            "type": "path",
+                            "expr": "$.fields.out_bytes"
+                        },
+                        {
+                            "name": "hit_count",
+                            "type": "path",
+                            "expr": "$.fields.hit_count"
+                        },
+                        {
+                            "name": "in_pkts",
+                            "type": "path",
+                            "expr": "$.fields.in_pkts"
+                        },
+                        {
+                            "name": "out_pkts",
+                            "type": "path",
+                            "expr": "$.fields.out_pkts"
+                        }
+                    ]
+                }
+            }
+        },
+        "metricsSpec": [
+            {
+                "type": "longSum",
+                "name": "in_bytes",
+                "fieldName": "in_bytes"
+            },
+            {
+                "type": "longSum",
+                "name": "out_bytes",
+                "fieldName": "out_bytes"
+            },
+            {
+                "type": "longSum",
+                "name": "hit_count",
+                "fieldName": "hit_count"
+            },
+            {
+                "type": "longSum",
+                "name": "in_pkts",
+                "fieldName": "in_pkts"
+            },
+            {
+                "type": "longSum",
+                "name": "out_pkts",
+                "fieldName": "out_pkts"
+            }
+        ],
+        "granularitySpec": {
+            "type": "uniform",
+            "segmentGranularity": "DAY",
+            "queryGranularity": {
+                "type": "period",
+                "period": "PT1S"
+            },
+            "rollup": true
+        },
+        "transformSpec": {
+            "transforms": [
+                {"type": "expression", "name": "vsys_id", "expression": "nvl(vsys_id,1)"},
+                {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
+                {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
+                {"type": "expression", "name": "hit_count", "expression": "nvl(hit_count, 0)"},
+                {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
+                {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
+            ],
+            "filter": {
+                "type": "selector",
+                "dimension": "name",
+                "value": "statistics_rule_hits"
+            }
+        }
+    },
+    "tuningConfig": {
+        "type": "kafka",
+        "maxRowsPerSegment": 5000000,
+        "resetOffsetAutomatically": true,
+        "reportParseExceptions": false
+    },
+    "ioConfig": {
+        "topic": "POLICY-RULE-METRIC",
+        "taskCount": 1,
+        "replicas": 1,
+        "taskDuration": "PT1H",
+        "completionTimeout": "PT30M",
+        "consumerProperties": {
+            "bootstrap.servers": "kafkabootstrap",
+            "sasl.mechanism": "PLAIN",
+            "security.protocol": "SASL_PLAINTEXT",
+            "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+        }
+    }
+}
+
+
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_sys_storage_log.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_sys_storage_log.json
new file mode 100644
index 0000000..2b405c2
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_sys_storage_log.json
@@ -0,0 +1,52 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "sys_storage_log",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "time",
+          "format": "posix"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+	   "log_type",
+	   "data_center",
+	   { "name": "used_size", "type": "long" },
+	   { "name": "max_size", "type": "long" },
+	   { "name": "aggregate_size", "type": "long" },
+	   { "name": "last_storage", "type": "long" }
+          ]
+        }
+      }
+    },
+    "metricsSpec" : [
+   ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT5M", "timeZone": "Asia/Hong_Kong"},
+      "rollup": false
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "SYS-STORAGE",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap",
+      "sasl.mechanism": "PLAIN",
+      "security.protocol": "SASL_PLAINTEXT",
+      "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+    }
+  }
+}
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_client_ips.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_client_ips.json
new file mode 100644
index 0000000..4a8749f
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_client_ips.json
@@ -0,0 +1,104 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "top_client_ips",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "timestamp_ms",
+          "format": "millis"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {"name": "vsys_id","type": "long"},
+	   "device_id",
+	   "device_group",
+           "data_center",
+           "client_ip",
+		   "metric"
+          ]
+        },
+	"flattenSpec": {
+  	  "useFieldDiscovery": true,
+  	  "fields": [
+                { "name": "sessions", "type": "path", "expr": "$.fields.sessions" },
+                { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" },
+                { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" },
+                { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" },
+                { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" },
+                { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" },
+                { "name": "client_ip", "type": "path", "expr": "$.tags.client_ip" },
+                { "name": "data_center", "type": "path", "expr": "$.tags.data_center" },
+                { "name": "device_group", "type": "path", "expr": "$.tags.device_group" },
+                { "name": "device_id", "type": "path", "expr": "$.tags.device_id" },
+		{ "name": "name", "type": "path", "expr": "$.name" }
+  	  ]
+	}
+      }
+    },
+    "metricsSpec" : [
+        { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" },
+        { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
+        { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
+        { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
+        { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
+  ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT1M"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[
+            {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"},
+            {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
+            {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
+            {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
+            {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
+            {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
+        ],
+       "filter":{
+         "type":"or","fields":[
+         {  
+               "type":"selector",
+               "dimension":"name",
+               "value":"packets_top_client_ip" 
+         },
+         {
+               "type":"selector",
+               "dimension":"name",
+               "value":"bytes_top_client_ip"
+        },
+	{	 
+               "type":"selector",
+               "dimension":"name",
+               "value":"sessions_top_client_ip" 
+        } 
+      ]
+   }
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "TRAFFIC-TOP-METRIC",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap",
+      "sasl.mechanism": "PLAIN",
+      "security.protocol": "SASL_PLAINTEXT",
+      "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+    }
+  }
+}
+
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_external_ips.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_external_ips.json
new file mode 100644
index 0000000..5ff580e
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_external_ips.json
@@ -0,0 +1,105 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "top_external_ips",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "timestamp_ms",
+          "format": "millis"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {"name": "vsys_id","type": "long"},
+	   "device_id",
+	   "device_group",
+           "data_center",
+           "external_ip",
+		   "metric"
+          ]
+        },
+	"flattenSpec": {
+  	  "useFieldDiscovery": true,
+  	  "fields": [
+                { "name": "sessions", "type": "path", "expr": "$.fields.sessions" },
+                { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" },
+                { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" },
+                { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" },
+                { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" },
+                { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" },
+                { "name": "external_ip", "type": "path", "expr": "$.tags.external_ip" },
+                { "name": "data_center", "type": "path", "expr": "$.tags.data_center" },
+                { "name": "device_group", "type": "path", "expr": "$.tags.device_group" },
+                { "name": "device_id", "type": "path", "expr": "$.tags.device_id" },
+		{ "name": "name", "type": "path", "expr": "$.name" }
+  	  ]
+	}
+      }
+    },
+    "metricsSpec" : [
+        { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" },
+        { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
+        { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
+        { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
+        { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
+  ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT1M"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[
+            {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"},
+            {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
+            {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
+            {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
+            {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
+            {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
+        ],
+       "filter":{
+         "type":"or","fields":[
+         {  
+               "type":"selector",
+               "dimension":"name",
+               "value":"sessions_top_external_ip" 
+         },
+         {
+               "type":"selector",
+               "dimension":"name",
+               "value":"bytes_top_external_ip"
+        },
+	{	 
+               "type":"selector",
+               "dimension":"name",
+               "value":"packets_top_external_ip" 
+        } 
+      ]
+   }
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "TRAFFIC-TOP-METRIC",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap",
+      "sasl.mechanism": "PLAIN",
+      "security.protocol": "SASL_PLAINTEXT",
+      "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+    }
+  }
+}
+
+
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_internal_ips.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_internal_ips.json
new file mode 100644
index 0000000..8767165
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_internal_ips.json
@@ -0,0 +1,105 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "top_internal_ips",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "timestamp_ms",
+          "format": "millis"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {"name": "vsys_id","type": "long"},
+	   "device_id",
+	   "device_group",
+           "data_center",
+           "internal_ip",
+		   "metric"
+          ]
+        },
+	"flattenSpec": {
+  	  "useFieldDiscovery": true,
+  	  "fields": [
+                { "name": "sessions", "type": "path", "expr": "$.fields.sessions" },
+                { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" },
+                { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" },
+                { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" },
+                { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" },
+                { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" },
+                { "name": "internal_ip", "type": "path", "expr": "$.tags.internal_ip" },
+                { "name": "data_center", "type": "path", "expr": "$.tags.data_center" },
+                { "name": "device_group", "type": "path", "expr": "$.tags.device_group" },
+                { "name": "device_id", "type": "path", "expr": "$.tags.device_id" },
+		{ "name": "name", "type": "path", "expr": "$.name" }
+  	  ]
+	}
+      }
+    },
+    "metricsSpec" : [
+        { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" },
+        { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
+        { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
+        { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
+        { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
+  ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT1M"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[
+            {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"},
+            {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
+            {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
+            {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
+            {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
+            {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
+        ],
+       "filter":{
+         "type":"or","fields":[
+         {  
+               "type":"selector",
+               "dimension":"name",
+               "value":"sessions_top_internal_ip" 
+         },
+         {
+               "type":"selector",
+               "dimension":"name",
+               "value":"bytes_top_internal_ip"
+        },
+	{	 
+               "type":"selector",
+               "dimension":"name",
+               "value":"packets_top_internal_ip" 
+        } 
+      ]
+   }
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "TRAFFIC-TOP-METRIC",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap",
+      "sasl.mechanism": "PLAIN",
+      "security.protocol": "SASL_PLAINTEXT",
+      "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+    }
+  }
+}
+
+
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_domains.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_domains.json
new file mode 100644
index 0000000..bd5b144
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_domains.json
@@ -0,0 +1,105 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "top_server_domains",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "timestamp_ms",
+          "format": "millis"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {"name": "vsys_id","type": "long"},
+	   "device_id",
+	   "device_group",
+           "data_center",
+           "domain",
+		   "metric"
+          ]
+        },
+	"flattenSpec": {
+  	  "useFieldDiscovery": true,
+  	  "fields": [
+                { "name": "sessions", "type": "path", "expr": "$.fields.sessions" },
+                { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" },
+                { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" },
+                { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" },
+                { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" },
+                { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" },
+                { "name": "domain", "type": "path", "expr": "$.tags.domain" },
+                { "name": "data_center", "type": "path", "expr": "$.tags.data_center" },
+                { "name": "device_group", "type": "path", "expr": "$.tags.device_group" },
+                { "name": "device_id", "type": "path", "expr": "$.tags.device_id" },
+		{ "name": "name", "type": "path", "expr": "$.name" }
+  	  ]
+	}
+      }
+    },
+    "metricsSpec" : [
+        { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" },
+        { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
+        { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
+        { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
+        { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
+  ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT1M"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[
+            {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"},
+            {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
+            {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
+            {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
+            {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
+            {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
+        ],
+       "filter":{
+         "type":"or","fields":[
+         {  
+               "type":"selector",
+               "dimension":"name",
+               "value":"packets_top_server_domain" 
+         },
+         {
+               "type":"selector",
+               "dimension":"name",
+               "value":"bytes_top_server_domain"
+        },
+	{	 
+               "type":"selector",
+               "dimension":"name",
+               "value":"sessions_top_server_domain" 
+        } 
+      ]
+   }
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "TRAFFIC-TOP-METRIC",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap",
+      "sasl.mechanism": "PLAIN",
+      "security.protocol": "SASL_PLAINTEXT",
+      "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+    }
+  }
+}
+
+
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_fqdns.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_fqdns.json
new file mode 100644
index 0000000..4bc42af
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_fqdns.json
@@ -0,0 +1,105 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "top_server_fqdns",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "timestamp_ms",
+          "format": "millis"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {"name": "vsys_id","type": "long"},
+	   "device_id",
+	   "device_group",
+           "data_center",
+           "fqdn",
+		   "metric"
+          ]
+        },
+	"flattenSpec": {
+  	  "useFieldDiscovery": true,
+  	  "fields": [
+                { "name": "sessions", "type": "path", "expr": "$.fields.sessions" },
+                { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" },
+                { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" },
+                { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" },
+                { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" },
+                { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" },
+                { "name": "fqdn", "type": "path", "expr": "$.tags.fqdn" },
+                { "name": "data_center", "type": "path", "expr": "$.tags.data_center" },
+                { "name": "device_group", "type": "path", "expr": "$.tags.device_group" },
+                { "name": "device_id", "type": "path", "expr": "$.tags.device_id" },
+		{ "name": "name", "type": "path", "expr": "$.name" }
+  	  ]
+	}
+      }
+    },
+    "metricsSpec" : [
+        { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" },
+        { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
+        { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
+        { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
+        { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
+  ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT1M"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[
+            {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"},
+            {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
+            {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
+            {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
+            {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
+            {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
+        ],
+       "filter":{
+         "type":"or","fields":[
+         {  
+               "type":"selector",
+               "dimension":"name",
+               "value":"sessions_top_server_fqdn" 
+         },
+         {
+               "type":"selector",
+               "dimension":"name",
+               "value":"bytes_top_server_fqdn"
+        },
+	{	 
+               "type":"selector",
+               "dimension":"name",
+               "value":"packets_top_server_fqdn" 
+        } 
+      ]
+   }
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "TRAFFIC-TOP-METRIC",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap",
+      "sasl.mechanism": "PLAIN",
+      "security.protocol": "SASL_PLAINTEXT",
+      "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+    }
+  }
+}
+
+
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_ips.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_ips.json
new file mode 100644
index 0000000..1f05adf
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_server_ips.json
@@ -0,0 +1,105 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "top_server_ips",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "timestamp_ms",
+          "format": "millis"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {"name": "vsys_id","type": "long"},
+	   "device_id",
+	   "device_group",
+           "data_center",
+           "server_ip",
+		   "metric"
+          ]
+        },
+	"flattenSpec": {
+  	  "useFieldDiscovery": true,
+  	  "fields": [
+                { "name": "sessions", "type": "path", "expr": "$.fields.sessions" },
+                { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" },
+                { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" },
+                { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" },
+                { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" },
+                { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" },
+                { "name": "server_ip", "type": "path", "expr": "$.tags.server_ip" },
+                { "name": "data_center", "type": "path", "expr": "$.tags.data_center" },
+                { "name": "device_group", "type": "path", "expr": "$.tags.device_group" },
+                { "name": "device_id", "type": "path", "expr": "$.tags.device_id" },
+		{ "name": "name", "type": "path", "expr": "$.name" }
+  	  ]
+	}
+      }
+    },
+    "metricsSpec" : [
+        { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" },
+        { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
+        { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
+        { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
+        { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
+  ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT1M"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[
+            {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"},
+            {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
+            {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
+            {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
+            {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
+            {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
+        ],
+       "filter":{
+         "type":"or","fields":[
+         {  
+               "type":"selector",
+               "dimension":"name",
+               "value":"sessions_top_server_ip" 
+         },
+         {
+               "type":"selector",
+               "dimension":"name",
+               "value":"bytes_top_server_ip"
+        },
+	{	 
+               "type":"selector",
+               "dimension":"name",
+               "value":"packets_top_server_ip" 
+        } 
+      ]
+   }
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "TRAFFIC-TOP-METRIC",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap",
+      "sasl.mechanism": "PLAIN",
+      "security.protocol": "SASL_PLAINTEXT",
+      "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+    }
+  }
+}
+
+
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_subscribers.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_subscribers.json
new file mode 100644
index 0000000..8bc5e37
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_top_subscribers.json
@@ -0,0 +1,105 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "top_subscribers",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "timestamp_ms",
+          "format": "millis"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {"name": "vsys_id","type": "long"},
+	   "device_id",
+	   "device_group",
+           "data_center",
+           "subscriber_id",
+		   "metric"
+          ]
+        },
+	"flattenSpec": {
+  	  "useFieldDiscovery": true,
+  	  "fields": [
+                { "name": "sessions", "type": "path", "expr": "$.fields.sessions" },
+                { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" },
+                { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" },
+                { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" },
+                { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" },
+                { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" },
+                { "name": "subscriber_id", "type": "path", "expr": "$.tags.subscriber_id" },
+                { "name": "data_center", "type": "path", "expr": "$.tags.data_center" },
+                { "name": "device_group", "type": "path", "expr": "$.tags.device_group" },
+                { "name": "device_id", "type": "path", "expr": "$.tags.device_id" },
+		{ "name": "name", "type": "path", "expr": "$.name" }
+  	  ]
+	}
+      }
+    },
+    "metricsSpec" : [
+        { "type" : "longSum", "name" : "sessions", "fieldName" : "sessions" },
+        { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
+        { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
+        { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
+        { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" }
+  ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT1M"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[
+            {"type": "expression", "name": "metric", "expression": "array_ordinal(string_to_array(name,'_'),1)"},
+            {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
+            {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
+            {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
+            {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
+            {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"}
+        ],
+       "filter":{
+         "type":"or","fields":[
+         {  
+               "type":"selector",
+               "dimension":"name",
+               "value":"sessions_top_subscriber_id" 
+         },
+         {
+               "type":"selector",
+               "dimension":"name",
+               "value":"bytes_top_subscriber_id"
+        },
+	{	 
+               "type":"selector",
+               "dimension":"name",
+               "value":"packets_top_subscriber_id" 
+        } 
+      ]
+   }
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "TRAFFIC-TOP-METRIC",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap",
+      "sasl.mechanism": "PLAIN",
+      "security.protocol": "SASL_PLAINTEXT",
+      "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+    }
+  }
+}
+
+
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_general_stat.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_general_stat.json
new file mode 100644
index 0000000..6013977
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_general_stat.json
@@ -0,0 +1,105 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "traffic_general_stat",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "timestamp_ms",
+          "format": "millis"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            "device_id",
+            "device_group",
+            "data_center",
+           { "name": "vsys_id", "type": "long" }
+          ]
+        },
+        "flattenSpec": {
+        "useFieldDiscovery": true,
+        "fields": [
+                { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" },
+                { "name": "device_id", "type": "path", "expr": "$.tags.device_id" },
+                { "name": "device_group", "type": "path", "expr": "$.tags.device_group" },        
+                { "name": "data_center", "type": "path", "expr": "$.tags.data_center" },
+                { "name": "sessions", "type": "path", "expr": "$.fields.sessions" },
+                { "name": "active_sessions", "type": "path", "expr": "$.fields.active_sessions" },
+                { "name": "closed_sessions", "type": "path", "expr": "$.fields.closed_sessions" },
+                { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" },
+                { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" },
+                { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" },
+                { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" },
+                { "name": "asymmetric_c2s_flows", "type": "path", "expr": "$.fields.asymmetric_c2s_flows" },
+                { "name": "asymmetric_s2c_flows", "type": "path", "expr": "$.fields.asymmetric_s2c_flows" },
+                { "name": "tcp_syn_pkts", "type": "path", "expr": "$.fields.tcp_syn_pkts" },
+                { "name": "tcp_ack_pkts", "type": "path", "expr": "$.fields.tcp_ack_pkts" },
+                { "name": "tcp_fin_pkts", "type": "path", "expr": "$.fields.tcp_fin_pkts" },
+                { "name": "tcp_rst_pkts", "type": "path", "expr": "$.fields.tcp_rst_pkts" }
+        ]
+    }
+      }
+    },
+        "metricsSpec":[
+            { "type":"longSum", "name":"sessions", "fieldName":"sessions" },
+            { "type":"longMax", "name":"active_sessions", "fieldName":"active_sessions" },
+            { "type":"longSum", "name":"closed_sessions", "fieldName":"closed_sessions" },
+            { "type":"longSum", "name":"in_bytes", "fieldName":"in_bytes" },
+            { "type":"longSum", "name":"out_bytes", "fieldName":"out_bytes" },
+            { "type":"longSum", "name":"in_pkts", "fieldName":"in_pkts" },
+            { "type":"longSum", "name":"out_pkts", "fieldName":"out_pkts" },
+            { "type":"longSum", "name":"asymmetric_c2s_flows", "fieldName":"asymmetric_c2s_flows" },
+            { "type":"longSum", "name":"asymmetric_s2c_flows", "fieldName":"asymmetric_s2c_flows" },
+            { "type":"longSum", "name":"tcp_syn_pkts", "fieldName":"tcp_syn_pkts" },
+            { "type":"longSum", "name":"tcp_ack_pkts", "fieldName":"tcp_ack_pkts" },
+            { "type":"longSum", "name":"tcp_fin_pkts", "fieldName":"tcp_fin_pkts" },
+            { "type":"longSum", "name":"tcp_rst_pkts", "fieldName":"tcp_rst_pkts" }
+        ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT1S"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[
+        {"type": "expression", "name": "sessions", "expression": "nvl(sessions, 0)"},
+        {"type": "expression", "name": "active_sessions", "expression": "nvl(active_sessions, 0)"},
+        {"type": "expression", "name": "closed_sessions", "expression": "nvl(closed_sessions, 0)"},
+        {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
+        {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
+        {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
+        {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"},
+        {"type": "expression", "name": "asymmetric_c2s_flows", "expression": "nvl(asymmetric_c2s_flows, 0)"},
+        {"type": "expression", "name": "asymmetric_s2c_flows", "expression": "nvl(asymmetric_s2c_flows, 0)"},
+        {"type": "expression", "name": "tcp_syn_pkts", "expression": "nvl(tcp_syn_pkts, 0)"},
+        {"type": "expression", "name": "tcp_ack_pkts", "expression": "nvl(tcp_ack_pkts, 0)"},
+        {"type": "expression", "name": "tcp_fin_pkts", "expression": "nvl(tcp_fin_pkts, 0)"},
+        {"type": "expression", "name": "tcp_rst_pkts", "expression": "nvl(tcp_rst_pkts, 0)"}
+      ],
+      "filter": { "type": "selector", "dimension": "name", "value": "traffic_general_stat" }
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "maxRowsPerSegment": 5000000,
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "NETWORK-TRAFFIC-METRIC",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap",
+      "sasl.mechanism": "PLAIN",
+      "security.protocol": "SASL_PLAINTEXT",
+      "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+    }
+  }
+}
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_shaping_rule_hits.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_shaping_rule_hits.json
new file mode 100644
index 0000000..a5a644d
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_shaping_rule_hits.json
@@ -0,0 +1,101 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "traffic_shaping_rule_hits",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "timestamp_ms",
+          "format": "millis"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {"name": "vsys_id","type": "long"},
+            "device_id",
+   	    "device_group",
+            "data_center",
+            {"name": "rule_id","type": "long"},
+            {"name": "profile_id","type": "long"},
+            {"name": "priority","type": "long"}
+          ]
+        },
+	"flattenSpec": {
+  	  "useFieldDiscovery": true,
+  	  "fields": [
+                { "name": "in_bytes", "type": "path", "expr": "$.fields.in_bytes" },
+                { "name": "out_bytes", "type": "path", "expr": "$.fields.out_bytes" },
+                { "name": "in_pkts", "type": "path", "expr": "$.fields.in_pkts" },
+                { "name": "out_pkts", "type": "path", "expr": "$.fields.out_pkts" },
+                { "name": "in_drop_pkts", "type": "path", "expr": "$.fields.in_drop_pkts" },
+                { "name": "out_drop_pkts", "type": "path", "expr": "$.fields.out_drop_pkts" },
+                { "name": "in_max_latency_us", "type": "path", "expr": "$.fields.in_max_latency_us" },
+                { "name": "out_max_latency_us", "type": "path", "expr": "$.fields.out_max_latency_us" },
+                { "name": "in_queue_len", "type": "path", "expr": "$.fields.in_queue_len" },
+                { "name": "out_queue_len", "type": "path", "expr": "$.fields.out_queue_len" },
+                { "name": "vsys_id", "type": "path", "expr": "$.tags.vsys_id" },
+                { "name": "data_center", "type": "path", "expr": "$.tags.data_center" },
+                { "name": "device_group", "type": "path", "expr": "$.tags.device_group" },
+                { "name": "device_id", "type": "path", "expr": "$.tags.device_id" },
+		{ "name": "rule_id", "type": "path", "expr": "$.tags.rule_id" },
+		{ "name": "profile_id", "type": "path", "expr": "$.tags.profile_id" },
+		{ "name": "priority", "type": "path", "expr": "$.tags.priority" }
+  	  ]
+	}
+      }
+    },
+    "metricsSpec" : [
+        { "type" : "longSum", "name" : "in_bytes", "fieldName" : "in_bytes" },
+        { "type" : "longSum", "name" : "out_bytes", "fieldName" : "out_bytes" },
+        { "type" : "longSum", "name" : "in_pkts", "fieldName" : "in_pkts" },
+        { "type" : "longSum", "name" : "out_pkts", "fieldName" : "out_pkts" },
+        { "type" : "longSum", "name" : "in_drop_pkts", "fieldName" : "in_drop_pkts" },
+        { "type" : "longSum", "name" : "out_drop_pkts", "fieldName" : "out_drop_pkts" },
+        { "type" : "longMax", "name" : "in_max_latency_us", "fieldName" : "in_max_latency_us" },
+        { "type" : "longMax", "name" : "out_max_latency_us", "fieldName" : "out_max_latency_us" },
+        { "type" : "longMax", "name" : "in_queue_len", "fieldName" : "in_queue_len" },
+        { "type" : "longMax", "name" : "out_queue_len", "fieldName" : "out_queue_len" }
+  ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT1S"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[
+        {"type": "expression", "name": "in_bytes", "expression": "nvl(in_bytes, 0)"},
+        {"type": "expression", "name": "out_bytes", "expression": "nvl(out_bytes, 0)"},
+        {"type": "expression", "name": "in_pkts", "expression": "nvl(in_pkts, 0)"},
+        {"type": "expression", "name": "out_pkts", "expression": "nvl(out_pkts, 0)"},
+        {"type": "expression", "name": "in_drop_pkts", "expression": "nvl(in_drop_pkts, 0)"},
+        {"type": "expression", "name": "out_drop_pkts", "expression": "nvl(out_drop_pkts, 0)"},
+        {"type": "expression", "name": "in_max_latency_us", "expression": "nvl(in_max_latency_us, 0)"},
+        {"type": "expression", "name": "out_max_latency_us", "expression": "nvl(out_max_latency_us, 0)"},
+        {"type": "expression", "name": "in_queue_len", "expression": "nvl(in_queue_len, 0)"},
+        {"type": "expression", "name": "out_queue_len", "expression": "nvl(out_queue_len, 0)"}
+      ],
+      "filter": { "type": "selector", "dimension": "name", "value": "traffic_shaping_rule_hits" }
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "POLICY-RULE-METRIC",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap",
+      "sasl.mechanism": "PLAIN",
+      "security.protocol": "SASL_PLAINTEXT",
+      "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+    }
+  }
+}
+
diff --git a/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_top_destination_ip_metrics_log.json b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_top_destination_ip_metrics_log.json
new file mode 100644
index 0000000..5541452
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/templates/tasks/index_kafka_traffic_top_destination_ip_metrics_log.json
@@ -0,0 +1,61 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "traffic_top_destination_ip_metrics_log",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "sketch_start_time",
+          "format": "posix"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+	  "destination_ip",
+	  "attack_type",
+           { "name": "vsys_id", "type": "long" },
+	  { "type": "long", "name": "partition_num" }
+          ]
+        }
+      }
+    },
+    "metricsSpec" : [
+	{ "type" : "longSum", "name" : "session_rate","fieldName" : "session_rate"},
+        { "type" : "longSum", "name" : "packet_rate", "fieldName" : "packet_rate" },
+        { "type" : "longSum", "name" : "bit_rate", "fieldName" : "bit_rate" }
+  ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": {"type": "period", "period": "PT10M", "timeZone": "Asia/Hong_Kong"},
+      "rollup": true
+    },
+    "transformSpec" :{
+      "transforms":[
+        {"type": "expression","name": "vsys_id","expression": "nvl(vsys_id,1)"},
+        {"type": "expression", "name": "session_rate", "expression": "nvl(session_rate, 0)"},
+        {"type": "expression", "name": "packet_rate", "expression": "nvl(packet_rate, 0)"},
+        {"type": "expression", "name": "bit_rate", "expression": "nvl(bit_rate, 0)"}
+      ]
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "resetOffsetAutomatically": true,
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "TRAFFIC-TOP-DESTINATION-IP-METRIC",
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT1H",
+    "completionTimeout": "PT30M",
+    "consumerProperties": {
+      "bootstrap.servers": "kafkabootstrap",
+      "sasl.mechanism": "PLAIN",
+      "security.protocol": "SASL_PLAINTEXT",
+      "sasl.jaas.config": "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"admin\" password=\"galaxy2019\";"
+    }
+  }
+}
diff --git a/tsg-olap/parcels/roles/initialization/druid/vars/main.yml b/tsg-olap/parcels/roles/initialization/druid/vars/main.yml
new file mode 100644
index 0000000..3fc948a
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/druid/vars/main.yml
@@ -0,0 +1,11 @@
+container_name: druid
+
+backup_path: "{{ deploy_dir }}/backup/tsg-olap/{{ old_version }}/{{ container_name }}"
+
+backup_items:
+  - conf
+  - docker-compose.yml
+  - init
+  - monitor
+  - topology
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/init.yml b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/init.yml
new file mode 100644
index 0000000..2c5336b
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/init.yml
@@ -0,0 +1,28 @@
+- name: Create buckets
+  uri:
+    url: "http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/{{ item }}"
+    method: PUT
+    headers:
+      token: "{{ hos_token }}"
+    return_content: yes
+  retries: 3
+  delay: 3
+  loop: "{{ init_buckets }}"
+  run_once: true
+
+- name: Check if buckets exist
+  uri:
+    url: "http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos/"
+    method: GET
+    headers:
+      token: "{{ hos_token }}"
+    return_content: yes
+  register: bucket_info
+  retries: 3
+  delay: 3
+
+- name: Check create bucket result
+  fail:
+    msg: "{{ item }} was not created successfully. Please retry."
+  when: "'{{ item }}' not in bucket_info.content"
+  loop: "{{ init_buckets }}"
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/main.yml
new file mode 100644
index 0000000..125d374
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/main.yml
@@ -0,0 +1,5 @@
+- include: init.yml
+  when: operation == 'install'
+
+- include: upgrade.yml
+  when: operation == 'upgrade'
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/upgrade.yml b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/upgrade.yml
new file mode 100644
index 0000000..cd21505
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/tasks/upgrade.yml
@@ -0,0 +1,2 @@
+---
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-hos-service/vars/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/vars/main.yml
new file mode 100644
index 0000000..9203abd
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-hos-service/vars/main.yml
@@ -0,0 +1,6 @@
+init_buckets:
+  - troubleshooting_file_bucket
+  - assessment_file_bucket
+  - knowledge_base_bucket
+  - traffic_file_bucket
+  - report_snapshot_bucket
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/handlers/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/handlers/main.yml
new file mode 100644
index 0000000..6df35ce
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/handlers/main.yml
@@ -0,0 +1,41 @@
+- name: Stop Container
+  docker_container:
+    name: '{{ image_name }}'
+    state: absent
+    force_kill: true
+
+- name: Remove Image
+  docker_image:
+    name: '{{ image_name }}'
+    tag: '{{ image_tag }}'
+    state: absent
+    force_absent: true
+
+- name: Copy Image
+  copy:
+    src: 'files/{{ image_name }}-{{ image_tag }}.tar'
+    dest: '{{ deploy_dir }}/{{ container_name }}'
+    force: yes
+
+- name: Unarchive Image
+  unarchive:
+    src: 'files/{{ image_name }}-{{ image_tag }}.tar.gz'
+    dest: '{{ deploy_dir }}/{{ container_name }}'
+    force: yes
+
+- name: Load Image
+  docker_image:
+    name: '{{ image_name }}'
+    tag: '{{ image_tag }}'
+    load_path: '{{ deploy_dir }}/{{ container_name }}/{{ image_name }}-{{ image_tag }}.tar'
+    source: load
+    force_tag: yes
+    force_source: yes
+    timeout: 300
+
+- name: Start Container
+  docker_compose:
+    project_src: '{{ deploy_dir }}/{{ container_name }}'
+
+- name: Start Nginx
+  shell: 'docker restart galaxy-gateway-nginx'
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/backup.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/backup.yml
new file mode 100644
index 0000000..3a66bc3
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/backup.yml
@@ -0,0 +1,51 @@
+- name: Check if admin backup directory exists
+  stat:
+    path: "{{ backup_path }}/{{ admin_container_name }}"
+  register: exist_status
+
+- name: Backup directories
+  block:
+    - name: Creat backup directory
+      file:
+        state: directory
+        path: "{{ backup_path }}/{{ admin_container_name }}"
+
+    - name: Copying directories to backup
+      copy:
+        src: "{{ deploy_dir }}/{{ admin_container_name }}/{{ item }}"
+        dest: "{{ backup_path }}/{{ admin_container_name }}/"
+        remote_src: true
+      loop: "{{ backup_items }}"
+
+  rescue:
+    - name: Remove backup directory on failure
+      file:
+        path: "{{ backup_path }}/{{ admin_container_name }}"
+        state: absent
+  when: exist_status.stat.exists == false
+
+- name: Check if executor backup directory exists
+  stat:
+    path: "{{ backup_path }}/{{ executor_container_name }}"
+  register: exist_status
+
+- name: Backup directories
+  block:
+    - name: Creat backup directory
+      file:
+        state: directory
+        path: "{{ backup_path }}/{{ executor_container_name }}"
+
+    - name: Copying directories to backup
+      copy:
+        src: "{{ deploy_dir }}/{{ executor_container_name }}/{{ item }}"
+        dest: "{{ backup_path }}/{{ executor_container_name }}/"
+        remote_src: true
+      loop: "{{ backup_items }}"
+
+  rescue:
+    - name: Remove backup directory on failure
+      file:
+        path: "{{ backup_path }}/{{ executor_container_name }}"
+        state: absent
+  when: exist_status.stat.exists == false
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/check.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/check.yml
new file mode 100644
index 0000000..14e1d40
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/check.yml
@@ -0,0 +1,12 @@
+- name: Check the health status of galaxy-job-admin
+  uri:
+    url: http://{{ vrrp_instance.default.virtual_ipaddress }}:8181/xxl-job-admin/login
+    method: POST
+    body_format: form-urlencoded
+    body:
+      userName: admin
+      pin: galaxy_2019
+
+- name: Check that the galaxy-job-admin is accessible
+  uri:
+    url: "http://{{ vrrp_instance.default.virtual_ipaddress }}:8181/xxl-job-admin/testCase/logPolicyTest"
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-admin.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-admin.yml
new file mode 100644
index 0000000..14cfe28
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-admin.yml
@@ -0,0 +1,52 @@
+- name: Set container_name
+  set_fact:
+    container_name: "{{ admin_container_name }}"    
+    image_name: "{{ admin_image_name }}"
+    image_tag: "{{ admin_image_tag }}"
+
+- name: Creating directory
+  file:
+    state: directory
+    path: '{{ deploy_dir }}/{{ container_name }}/config'
+
+- name: Copying image to {{ deploy_dir }}/{{ container_name }}/
+  copy:
+    src: '{{ software_packages_dir }}/{{ image_name }}-{{ image_tag }}.tar'
+    dest: '{{ deploy_dir }}/{{ container_name }}/'
+    force: true
+  notify:
+    - Load Image
+    - Start Container
+
+- name: Copying {{ container_name }} config
+  template:
+    src: "admin/{{ item }}.j2"
+    dest: '{{ deploy_dir }}/{{ container_name }}/config/{{ item }}'
+    force: yes
+  loop: 
+    - application.properties
+    - logback.xml
+  notify:
+    - Start Container
+
+- name: Copying {{ container_name }} docker-compose.yml
+  template:
+    src: "admin/docker-compose.yml.j2"
+    dest: '{{ deploy_dir }}/{{ container_name }}/docker-compose.yml'
+    force: yes
+  notify:
+    - Start Container
+
+- name: Copying {{ container_name }} nginx config
+  template:
+    src: "admin/{{ container_name }}.conf.j2"
+    dest: '{{ deploy_dir }}/galaxy-gateway-nginx/conf/includes/{{ container_name }}.conf'
+    force: yes
+  notify:
+    - Start Nginx
+  delegate_to: "{{ item }}"
+  run_once: true
+  loop: "{{ groups.loadbalancer[:2] }}"
+
+- meta: flush_handlers
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-executor.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-executor.yml
new file mode 100644
index 0000000..7ea6d93
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/deploy-executor.yml
@@ -0,0 +1,45 @@
+- name: Setting variable name
+  set_fact:
+    container_name: "{{ executor_container_name }}"    
+    image_name: "{{ executor_image_name }}"
+    image_tag: "{{ executor_image_tag }}"
+
+- name: Creating {{ container_name }} directory
+  file:
+    state: directory
+    path: '{{ deploy_dir }}/{{ container_name }}/{{ item }}'
+  loop: 
+    - "init"
+    - "config"
+
+- name: Copying image to {{ deploy_dir }}/{{ container_name }}/
+  copy:
+    src: '{{ software_packages_dir }}/{{ image_name }}-{{ image_tag }}.tar'
+    dest: '{{ deploy_dir }}/{{ container_name }}/'
+    force: true
+  notify:
+    - Load Image
+    - Start Container
+
+- name: Copying {{ container_name }} config
+  template:
+    src: "executor/{{ item }}.j2"
+    dest: '{{ deploy_dir }}/{{ container_name }}/config/{{ item }}'
+    force: yes
+  loop: 
+    - application.properties
+    - logback.xml
+  notify:
+    - Start Container
+
+- name: Copying {{ container_name }} docker-compose.yml
+  template:
+    src: "executor/docker-compose.yml.j2"
+    dest: '{{ deploy_dir }}/{{ container_name }}/docker-compose.yml'
+    force: yes
+  notify:
+    - Start Container
+
+
+- meta: flush_handlers
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/init.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/init.yml
new file mode 100644
index 0000000..d3385f7
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/init.yml
@@ -0,0 +1,53 @@
+- name: Creating directory
+  file:
+    state: directory
+    path: '{{ deploy_dir }}/{{ item }}/init'
+  loop:
+    - 'galaxy-job-admin'
+    - 'galaxy-job-executor'
+
+- name: Copying admin init files
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ deploy_dir }}/{{ admin_container_name }}/init/{{ item.dest }}"
+    mode: "{{ item.mode }}"
+    force: true
+  loop:
+    - { src: "push_config.sh.j2", dest: "push_config.sh", mode: "0755" }
+    - { src: "admin/xxl_job.sql.j2", dest: "xxl_job.sql", mode: "0644" }
+    - { src: 'admin/create_mariadb_tables.sh.j2', dest: 'create_mariadb_tables.sh', mode: "0755" }
+    - { src: "admin/galaxy-job-admin.properties.j2", dest: "galaxy-job-admin.properties", mode: "0644" }  
+
+- name: Copying mysql Client to /usr/bin/
+  copy:
+    src: 'files/mysql'
+    dest: '/usr/bin/'
+    mode: 0755
+    force: true
+
+- name: Creating table and init jobs
+  shell: cd {{ deploy_dir }}/{{ admin_container_name }}/init && sh create_mariadb_tables.sh
+  run_once: true
+  when: operation == 'install'
+
+- name: Copying executor init files
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ deploy_dir }}/{{ executor_container_name }}/init/{{ item.dest }}"
+    mode: "{{ item.mode }}"
+    force: true
+  loop:
+    - { src: "push_config.sh.j2", dest: "push_config.sh", mode: "0755" }
+    - { src: "executor/galaxy-job-executor.properties.j2", dest: "galaxy-job-executor.properties", mode: "0644" }
+
+- name: Pushing Galaxy-job-service admin config
+  shell: 'cd {{ deploy_dir }}/{{ item.src }}/init && sh push_config.sh {{ item.conf }}'
+  register: post_result
+  until: post_result.stdout == '1'
+  retries: 3
+  delay: 3
+  failed_when: post_result.stdout != '1'
+  loop: 
+    - { src: "{{ admin_container_name }}" , conf: "galaxy-job-admin.properties" }
+    - { src: "{{ executor_container_name }}" , conf: "galaxy-job-executor.properties" }
+  run_once: true
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/main.yml
new file mode 100644
index 0000000..a1d5b98
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/main.yml
@@ -0,0 +1,22 @@
+- block:
+  - include: uninstall-admin.yml
+  - include: uninstall-executor.yml
+  - include: init.yml
+  - include: deploy-admin.yml
+  - include: deploy-executor.yml
+  when: operation == 'install'
+
+- block:
+  - include: check.yml
+  - include: backup.yml
+  - include: init.yml
+  - include: upgrade.yml
+  - include: deploy-executor.yml
+  - include: check.yml
+  when: operation == 'upgrade'
+
+- block:
+  - include: uninstall-admin.yml
+  - include: uninstall-executor.yml
+  when: operation == 'uninstall'
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-admin.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-admin.yml
new file mode 100644
index 0000000..2ebe9c8
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-admin.yml
@@ -0,0 +1,18 @@
+- name: Stopping and removing {{ admin_container_name }} container
+  docker_container:
+    name: '{{ admin_container_name }}'
+    state: absent
+    force_kill: true
+
+- name: Removing old {{ admin_image_name }} image
+  docker_image:
+    name: '{{ admin_image_name }}'
+    tag: '{{ admin_image_tag }}'
+    state: absent
+    force_absent: true
+
+- name: Ansible delete old {{ deploy_dir }}/{{ admin_container_name }}
+  file:
+    path: '{{ deploy_dir }}/{{ admin_container_name }}'
+    state: absent
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-executor.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-executor.yml
new file mode 100644
index 0000000..3e4deb8
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/uninstall-executor.yml
@@ -0,0 +1,18 @@
+- name: Stopping and removing {{ executor_container_name }} container
+  docker_container:
+    name: '{{ executor_container_name }}'
+    state: absent
+    force_kill: true
+
+- name: Removing old {{ executor_image_name }} image
+  docker_image:
+    name: '{{ executor_image_name }}'
+    tag: '{{ executor_image_tag }}'
+    state: absent
+    force_absent: true
+
+- name: Ansible delete old {{ deploy_dir }}/{{ executor_container_name }}
+  file:
+    path: '{{ deploy_dir }}/{{ executor_container_name }}'
+    state: absent
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/upgrade.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/upgrade.yml
new file mode 100644
index 0000000..5da8917
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/tasks/upgrade.yml
@@ -0,0 +1,54 @@
+- name: Creating config tmp directory
+  file:
+    state: directory
+    path: "/tmp/{{ item }}"
+  loop: 
+    - "{{ admin_container_name }}"
+    - "{{ executor_container_name }}"
+
+- name: Creating upgrade directory
+  file:
+    state: directory
+    path: "{{ deploy_dir }}/{{ executor_container_name }}/upgrade/upgrade-sql"
+
+- name: Fetching config files
+  fetch:
+    src: "{{ deploy_dir }}/{{ item.name }}/{{ item.config }}"
+    dest: "/tmp/{{ item.name }}/{{ item.config }}"
+    flat: yes
+  loop:
+    - { name: "{{ admin_container_name }}", config: 'docker-compose.yml' }
+    - { name: "{{ executor_container_name }}", config: 'docker-compose.yml' }
+
+- name: Loading variable by admin docker-compose.yml
+  include_vars:
+    file: "/tmp/{{ admin_container_name }}/docker-compose.yml"
+    name: admin_docker_config
+
+- name: Loading variable by executor docker-compose.yml
+  include_vars:
+    file: "/tmp/{{ executor_container_name }}/docker-compose.yml"
+    name: executor_docker_config
+
+- name: Set variable 
+  set_fact:
+    galaxy_job_service:
+      admin_java_opts: "{{ admin_docker_config['services']['galaxy-job-admin']['environment']['JAVA_OPTS'] }}"
+      executor_java_opts: "{{ executor_docker_config['services']['galaxy-job-executor']['environment']['JAVA_OPTS'] }}"
+
+- name: Copying upgrade sql file
+  copy:
+    src: 'files/upgrade-sql'
+    dest: '{{ deploy_dir }}/{{ executor_container_name }}/upgrade/'
+    force: true
+
+- name: Copying upgrade-mariadb-data.sh template
+  template:
+    src: "upgrade-mariadb-data.sh.j2"
+    dest: "{{ deploy_dir }}/{{ executor_container_name }}/upgrade/upgrade-mariadb-data.sh"
+    mode: 0755
+    force: true
+
+- name: Upgrade mariadb data 
+  shell: cd {{ deploy_dir }}/{{ executor_container_name }}/upgrade && sh upgrade-mariadb-data.sh
+  run_once: true
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/application.properties.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/application.properties.j2
new file mode 100644
index 0000000..6ea1881
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/application.properties.j2
@@ -0,0 +1,23 @@
+#配置集的配置格式
+nacos.config.type=properties
+#配置中心地址
+nacos.config.server-addr={{ vrrp_instance.default.virtual_ipaddress }}:8848
+#命名空间
+nacos.config.namespace={{ config_namespace }}
+#数据集ID
+nacos.config.data-id=galaxy-job-admin.properties
+#开启自动刷新
+nacos.config.auto-refresh=true
+#配置对应的分组
+nacos.config.group=Galaxy
+#Nacos认证用户
+nacos.config.username=nacos
+#Nacos认证密码
+nacos.config.password=nacos
+#开启配置预加载功能
+nacos.config.bootstrap.enable=true
+#开启Nacos支持日志级别的加载时机
+nacos.config.bootstrap.log-enable=true
+
+## log config
+logging.config=./config/logback.xml
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/create_mariadb_tables.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/create_mariadb_tables.sh.j2
new file mode 100644
index 0000000..392997f
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/create_mariadb_tables.sh.j2
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ vrrp_instance.default.virtual_ipaddress }} -e "create database if not exists {{ mariadb_job_database }} default character set utf8mb4 collate utf8mb4_general_ci;"
+
+mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ vrrp_instance.default.virtual_ipaddress }} {{ mariadb_job_database }} < {{ deploy_dir }}/{{ admin_container_name }}/init/xxl_job.sql
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/docker-compose.yml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/docker-compose.yml.j2
new file mode 100644
index 0000000..be4abb8
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/docker-compose.yml.j2
@@ -0,0 +1,15 @@
+version: '3'
+services:
+  galaxy-job-admin:
+    image: {{ image_name }}:{{ image_tag }}
+    container_name: galaxy-job-admin
+    environment:
+      JAVA_OPTS: "{{ galaxy_job_service.admin_java_opts }}"
+    ports:
+      - "8181:8181"
+    volumes:
+      - "{{ deploy_dir }}/{{ container_name }}/logs:/logs"
+      - "{{ deploy_dir }}/{{ container_name }}/data:/data"
+      - "{{ deploy_dir }}/{{ container_name }}/config:/home/tsg/galaxy/galaxy-job/galaxy-job-admin/config"
+    restart: always
+    network_mode: "host"
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.conf.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.conf.j2
new file mode 100644
index 0000000..bee9eea
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.conf.j2
@@ -0,0 +1,16 @@
+upstream jobAdmin {
+
+{% for host in groups['galaxy_job_service'][:2] %}
+    server {{ host }}:8184;
+{% endfor %}
+
+}
+
+server {
+     listen  8181;
+     server_name  localhost;
+     location / {
+         proxy_pass http://jobAdmin;
+    }
+}
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.properties.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.properties.j2
new file mode 100644
index 0000000..082e719
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/galaxy-job-admin.properties.j2
@@ -0,0 +1,73 @@
+### web
+server.port=8184
+server.servlet.context-path=/xxl-job-admin
+spring.application.name=galaxy-job-admin
+### actuator
+management.server.servlet.context-path=/actuator
+management.health.mail.enabled=false
+management.endpoints.web.exposure.include=*
+#详细的应用健康信息 prometheus
+management.endpoint.health.show-details=always
+
+management.endpoint.metrics.enabled=true
+management.endpoint.prometheus.enabled=true
+management.metrics.export.prometheus.enabled=true
+management.metrics.tags.application=${spring.application.name}
+management.metrics.tags.module=${spring.application.name}
+
+
+### resources
+spring.mvc.servlet.load-on-startup=0
+spring.mvc.static-path-pattern=/static/**
+spring.resources.static-locations=classpath:/static/
+
+### freemarker
+spring.freemarker.templateLoaderPath=classpath:/templates/
+spring.freemarker.suffix=.ftl
+spring.freemarker.charset=UTF-8
+spring.freemarker.request-context-attribute=request
+spring.freemarker.settings.number_format=0.##########
+
+### mybatis
+mybatis.mapper-locations=classpath:/mybatis-mapper/*Mapper.xml
+#mybatis.type-aliases-package=com.xxl.job.admin.core.model
+
+### xxl-job, datasource
+spring.datasource.url=jdbc:mysql://{{ vrrp_instance.default.virtual_ipaddress }}:3306/{{ mariadb_job_database }}?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false&serverTimezone=UTC
+spring.datasource.username=ENC(63aTpwv2vH0vPikW+3Jjig==)
+spring.datasource.password=ENC(LDEb2OekU7iZWiFw6pUYBSozVKP27r1y)
+spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver
+
+### datasource-pool
+spring.datasource.type=com.zaxxer.hikari.HikariDataSource
+spring.datasource.hikari.minimum-idle=10
+spring.datasource.hikari.maximum-pool-size=100
+spring.datasource.hikari.auto-commit=true
+spring.datasource.hikari.idle-timeout=30000
+spring.datasource.hikari.pool-name=HikariCP
+spring.datasource.hikari.max-lifetime=900000
+spring.datasource.hikari.connection-timeout=30000
+spring.datasource.hikari.connection-test-query=SELECT 1
+
+### xxl-job, email
+spring.mail.host=smtp.qq.com
+spring.mail.port=25
[email protected]
+spring.mail.password=xxx
+spring.mail.properties.mail.smtp.auth=true
+spring.mail.properties.mail.smtp.starttls.enable=true
+spring.mail.properties.mail.smtp.starttls.required=true
+spring.mail.properties.mail.smtp.socketFactory.class=javax.net.ssl.SSLSocketFactory
+
+### xxl-job, access token
+xxl.job.accessToken=
+
+### xxl-job, i18n (default is zh_CN, and you can choose "zh_CN", "zh_TC" and "en")
+xxl.job.i18n=zh_CN
+
+## xxl-job, triggerpool max size
+xxl.job.triggerpool.fast.max=200
+xxl.job.triggerpool.slow.max=100
+
+### xxl-job, log retention days
+xxl.job.logretentiondays=30
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/logback.xml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/logback.xml.j2
new file mode 100644
index 0000000..dd729b0
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/logback.xml.j2
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration scan="true" scanPeriod="60 seconds" debug="false">
+
+    <!--每100M压缩日志文件-->
+    <property name="LOG_SIZE" value="100MB"/>
+    <!--日志文件路径-->
+    <property name="LOG_PATH" value="/logs"/>
+    <!--日志文件名称-->
+    <property name="LOG_FILE_NAME" value="galaxy-job-admin"/>
+    <!--日志打印等级-->
+    <property name="LOG_LEVEL" value="error"/>
+    <!--日志最大的历史30天 -->
+    <property name="LOG_DAYS" value="30"/>
+    <!--日志打印格式-->
+    <property name="LOG_PATTERN" value="%d{yyyy-MM-dd HH:mm:ssZ,UTC} [%thread] %-5level %logger -%msg%n"/>
+
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>${LOG_PATTERN}</pattern>
+        </encoder>
+    </appender>
+  
+    <appender name="ALL" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${LOG_PATH}/${LOG_FILE_NAME}.log</file>
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>ALL</level>
+        </filter>
+        <encoder>
+            <pattern>${LOG_PATTERN}</pattern>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>
+                ${LOG_PATH}/history/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz
+            </fileNamePattern>
+            <maxHistory>${LOG_DAYS}</maxHistory>
+            <maxFileSize>${LOG_SIZE}</maxFileSize>
+        </rollingPolicy>
+    </appender>
+
+    <root level="${LOG_LEVEL}">
+        <appender-ref ref="ALL"/>
+        <appender-ref ref="STDOUT"/>
+    </root>
+
+</configuration>
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/xxl_job.sql.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/xxl_job.sql.j2
new file mode 100644
index 0000000..f3562c8
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/admin/xxl_job.sql.j2
@@ -0,0 +1,195 @@
+/*
+Navicat MySQL Data Transfer
+
+Target Server Type    : MYSQL
+Target Server Version : 50505
+File Encoding         : 65001
+
+Date: 2021-09-23 14:05:11
+*/
+
+SET FOREIGN_KEY_CHECKS=0;
+
+-- ----------------------------
+-- Table structure for `xxl_job_group`
+-- ----------------------------
+DROP TABLE IF EXISTS `xxl_job_group`;
+CREATE TABLE `xxl_job_group` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `app_name` varchar(64) NOT NULL COMMENT '执行器AppName',
+  `title` varchar(64) NOT NULL COMMENT '执行器名称',
+  `address_type` tinyint(4) NOT NULL DEFAULT 0 COMMENT '执行器地址类型：0=自动注册、1=手动录入',
+  `address_list` varchar(512) DEFAULT NULL COMMENT '执行器地址列表，多地址逗号分隔',
+  PRIMARY KEY (`id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC;
+
+-- ----------------------------
+-- Table structure for `xxl_job_lock`
+-- ----------------------------
+DROP TABLE IF EXISTS `xxl_job_lock`;
+CREATE TABLE `xxl_job_lock` (
+  `lock_name` varchar(50) NOT NULL COMMENT '锁名称',
+  PRIMARY KEY (`lock_name`) USING BTREE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC;
+
+-- ----------------------------
+-- Records of xxl_job_lock
+-- ----------------------------
+INSERT INTO `xxl_job_lock` VALUES ('schedule_lock');
+
+-- ----------------------------
+-- Table structure for `xxl_job_log`
+-- ----------------------------
+DROP TABLE IF EXISTS `xxl_job_log`;
+CREATE TABLE `xxl_job_log` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `job_group` int(11) NOT NULL COMMENT '执行器主键ID',
+  `job_id` int(11) NOT NULL COMMENT '任务，主键ID',
+  `executor_address` varchar(255) DEFAULT NULL COMMENT '执行器地址，本次执行的地址',
+  `executor_handler` varchar(255) DEFAULT NULL COMMENT '执行器任务handler',
+  `executor_param` varchar(4000) DEFAULT '' COMMENT '执行器任务参数',
+  `executor_sharding_param` varchar(20) DEFAULT NULL COMMENT '执行器任务分片参数，格式如 1/2',
+  `executor_fail_retry_count` int(11) NOT NULL DEFAULT 0 COMMENT '失败重试次数',
+  `trigger_time` datetime DEFAULT NULL COMMENT '调度-时间',
+  `trigger_code` int(11) NOT NULL COMMENT '调度-结果',
+  `trigger_msg` text DEFAULT NULL COMMENT '调度-日志',
+  `handle_time` datetime DEFAULT NULL COMMENT '执行-时间',
+  `handle_code` int(11) NOT NULL COMMENT '执行-状态',
+  `handle_msg` text DEFAULT NULL COMMENT '执行-日志',
+  `alarm_status` tinyint(4) NOT NULL DEFAULT 0 COMMENT '告警状态：0-默认、1-无需告警、2-告警成功、3-告警失败',
+  PRIMARY KEY (`id`) USING BTREE,
+  KEY `I_trigger_time` (`trigger_time`) USING BTREE,
+  KEY `I_handle_code` (`handle_code`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=2077229 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC;
+
+-- ----------------------------
+-- Records of xxl_job_log
+-- ----------------------------
+
+-- ----------------------------
+-- Table structure for `xxl_job_logglue`
+-- ----------------------------
+DROP TABLE IF EXISTS `xxl_job_logglue`;
+CREATE TABLE `xxl_job_logglue` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `job_id` int(11) NOT NULL COMMENT '任务，主键ID',
+  `glue_type` varchar(50) DEFAULT NULL COMMENT 'GLUE类型',
+  `glue_source` mediumtext DEFAULT NULL COMMENT 'GLUE源代码',
+  `glue_remark` varchar(128) NOT NULL COMMENT 'GLUE备注',
+  `add_time` datetime DEFAULT NULL,
+  `update_time` datetime DEFAULT NULL,
+  PRIMARY KEY (`id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC;
+
+-- ----------------------------
+-- Records of xxl_job_logglue
+-- ----------------------------
+
+-- ----------------------------
+-- Table structure for `xxl_job_log_report`
+-- ----------------------------
+DROP TABLE IF EXISTS `xxl_job_log_report`;
+CREATE TABLE `xxl_job_log_report` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `trigger_day` datetime DEFAULT NULL COMMENT '调度-时间',
+  `running_count` int(11) NOT NULL DEFAULT 0 COMMENT '运行中-日志数量',
+  `suc_count` int(11) NOT NULL DEFAULT 0 COMMENT '执行成功-日志数量',
+  `fail_count` int(11) NOT NULL DEFAULT 0 COMMENT '执行失败-日志数量',
+  PRIMARY KEY (`id`) USING BTREE,
+  UNIQUE KEY `i_trigger_day` (`trigger_day`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=786 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC;
+
+-- ----------------------------
+-- Records of xxl_job_log_report
+-- ----------------------------
+
+-- ----------------------------
+-- Table structure for `xxl_job_registry`
+-- ----------------------------
+DROP TABLE IF EXISTS `xxl_job_registry`;
+CREATE TABLE `xxl_job_registry` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `registry_group` varchar(50) NOT NULL,
+  `registry_key` varchar(255) NOT NULL,
+  `registry_value` varchar(255) NOT NULL,
+  `update_time` datetime DEFAULT NULL,
+  PRIMARY KEY (`id`) USING BTREE,
+  KEY `i_g_k_v` (`registry_group`,`registry_key`,`registry_value`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=276 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC;
+
+-- ----------------------------
+-- Table structure for `xxl_job_user`
+-- ----------------------------
+DROP TABLE IF EXISTS `xxl_job_user`;
+CREATE TABLE `xxl_job_user` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `username` varchar(50) NOT NULL COMMENT '账号',
+  `password` varchar(50) NOT NULL COMMENT '密码',
+  `role` tinyint(4) NOT NULL COMMENT '角色：0-普通用户、1-管理员',
+  `permission` varchar(255) DEFAULT NULL COMMENT '权限：执行器ID列表，多个逗号分割',
+  PRIMARY KEY (`id`) USING BTREE,
+  UNIQUE KEY `i_username` (`username`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC;
+
+-- ----------------------------
+-- Records of xxl_job_user
+-- ----------------------------
+INSERT INTO `xxl_job_user` VALUES ('1', 'admin', 'fea191a3fdd9f68503f9fa0e8d0293ab', '1', null);
+INSERT INTO `xxl_job_user` VALUES ('2', 'query', '95497ea23bf8d27d86526717578e366f', '0', '3');
+
+-- ----------------------------
+-- Table structure for `xxl_job_info`
+-- ----------------------------
+DROP TABLE IF EXISTS `xxl_job_info`;
+CREATE TABLE `xxl_job_info` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `job_group` int(11) NOT NULL COMMENT '执行器主键ID',
+  `job_cron` varchar(128) NOT NULL COMMENT '任务执行CRON',
+  `job_desc` varchar(255) NOT NULL,
+  `add_time` datetime DEFAULT NULL,
+  `update_time` datetime DEFAULT NULL,
+  `author` varchar(64) DEFAULT NULL COMMENT '作者',
+  `alarm_email` varchar(255) DEFAULT NULL COMMENT '报警邮件',
+  `executor_route_strategy` varchar(50) DEFAULT NULL COMMENT '执行器路由策略',
+  `executor_handler` varchar(255) DEFAULT NULL COMMENT '执行器任务handler',
+  `executor_param` varchar(4000) DEFAULT '' COMMENT '执行器任务参数',
+  `executor_block_strategy` varchar(50) DEFAULT NULL COMMENT '阻塞处理策略',
+  `executor_timeout` int(11) NOT NULL DEFAULT 0 COMMENT '任务执行超时时间，单位秒',
+  `executor_fail_retry_count` int(11) NOT NULL DEFAULT 0 COMMENT '失败重试次数',
+  `glue_type` varchar(50) NOT NULL COMMENT 'GLUE类型',
+  `glue_source` mediumtext DEFAULT NULL COMMENT 'GLUE源代码',
+  `glue_remark` varchar(128) DEFAULT NULL COMMENT 'GLUE备注',
+  `glue_updatetime` datetime DEFAULT NULL COMMENT 'GLUE更新时间',
+  `child_jobid` varchar(255) DEFAULT NULL COMMENT '子任务ID，多个逗号分隔',
+  `trigger_status` tinyint(4) NOT NULL DEFAULT 0 COMMENT '调度状态：0-停止，1-运行',
+  `trigger_last_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '上次调度时间',
+  `trigger_next_time` bigint(13) NOT NULL DEFAULT 0 COMMENT '下次调度时间',
+  PRIMARY KEY (`id`) USING BTREE
+) ENGINE=InnoDB AUTO_INCREMENT=88 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC;
+
+
+-- ----------------------------
+-- Records of xxl_job_group
+-- ----------------------------
+INSERT INTO `xxl_job_group` VALUES (1, 'galaxy-executor', 'GDP', 1, '{{ job_executor_servers }}');
+INSERT INTO `xxl_job_group` VALUES (2, 'galaxy-executor', 'TSG_OLAP', 1, '{{ job_executor_servers }}');
+-- ----------------------------
+-- Records of xxl_job_info
+-- ----------------------------
+INSERT INTO `xxl_job_info` VALUES (58, 1, '0 0 0 1/1 * ? ', 'set_druid_table_ttl', '2020-07-30 10:34:34', '2023-12-23 11:39:23', 'zhq', '', 'FAILOVER', 'deleteReportAndMetricsDataJobHandler', '{\"defaultMaxDays\":365,\"maxDays\":365}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 10:34:34', '', 1, 1704326400000, 1704412800000);
+INSERT INTO `xxl_job_info` VALUES (59, 1, '0 0 0 1 * ?', 'clear_druid_table', '2020-07-30 11:32:08', '2022-04-12 02:38:39', 'zhq', '', 'FAILOVER', 'deleteAllReportAndMetricsDataJobHandler', '{\"maxDays\":88,\"defaultMaxDays\":365}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:32:08', '', 0, 0, 0);
+INSERT INTO `xxl_job_info` VALUES (62, 1, '0 0 0 1/1 * ? ', 'set_hos_bucket_ttl', '2020-07-30 11:59:30', '2023-12-23 11:39:23', 'zhq', '', 'FAILOVER', 'deleteFilesJobHandler', '{\"defaultMaxDays\":30,\"maxDays\":365}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:59:30', '', 1, 1704326400000, 1704412800000);
+INSERT INTO `xxl_job_info` VALUES (63, 1, '0 0 0 1 * ?', 'clear_hos_file', '2020-07-30 11:59:43', '2023-10-26 10:43:46', 'zhq', '', 'FAILOVER', 'deleteAllFilesJobHandler', '{\"defaultMaxDays\":30,\"maxDays\":30}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 11:59:43', '', 0, 0, 0);
+INSERT INTO `xxl_job_info` VALUES (64, 1, '0 0 0 1 * ?', 'clear_clickhouse_table', '2020-07-30 13:47:25', '2022-04-12 02:38:39', 'zhq', '', 'FAILOVER', 'deleteAllTrafficDataJobHandler', '{\"maxDays\":77,\"defaultMaxDays\":30}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 13:47:25', '', 0, 0, 0);
+INSERT INTO `xxl_job_info` VALUES (65, 1, '0 0 0 1/1 * ? ', 'set_clickhouse_table_ttl', '2020-07-30 13:47:38', '2023-12-23 11:39:23', 'zhq', '', 'FAILOVER', 'deleteTrafficDataJobHandler', '{\"defaultMaxDays\":30,\"maxDays\":30}', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-07-30 13:47:38', '', 1, 1704326400000, 1704412800000);
+
+INSERT INTO `xxl_job_info` VALUES (29, 2, '0 0 4 * * ?', 'get_clickhoue_storage_daily_usage', '2020-03-26 10:58:09', '2021-09-09 11:54:53', 'zhq', '', 'FAILOVER', 'getStorageQuotaJobHandler', '{\"topic\":\"SYS-STORAGE\",\"ckDayGrowth\":\"true\"}', 'SERIAL_EXECUTION', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-03-26 10:58:09', '', 1, 1704254400000, 1704340800000);
+INSERT INTO `xxl_job_info` VALUES (36, 2, '0 0/5 * * * ?', 'get_database_storage_quota', '2020-04-07 21:41:20', '2021-09-09 11:54:44', 'zhq', '', 'FAILOVER', 'getStorageQuotaJobHandler', '{\"topic\":\"SYS-STORAGE\",\"ckDayGrowth\":\"false\"}', 'SERIAL_EXECUTION', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2020-04-07 21:41:20', '', 1, 1704338100000, 1704338400000);
+INSERT INTO `xxl_job_info` VALUES (86, 2, '0 0 1 * * ?', 'run_druid_segment_compaction', '2021-07-21 14:13:57', '2023-12-12 02:21:05', 'xpf', '', 'FAILOVER', 'druidCompactSegmentJobHandler', '[\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"top_server_fqdns\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"top_server_domains\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"top_client_ips\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"top_server_ips\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"top_external_ips\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"top_internal_ips\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"top_subscribers\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"security_rule_hits\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"monitor_rule_hits\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"proxy_rule_hits\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"application_protocol_stat\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"traffic_general_stat\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"traffic_top_destination_ip_metrics_log\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"service_chaining_rule_hits\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"service_function_status\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"statistics_rule_hits\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"traffic_shaping_rule_hits\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"object_statistics\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"statistics_rule\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\",\r\n      \"maxRowsInMemory\": 500000,\r\n      \"maxBytesInMemory\": -1\r\n    },\r\n    \"context\":{\r\n      \"druid.indexer.runner.javaOptsArray\": [\"-Xms4g\", \"-Xmx4g\", \"-XX:MaxDirectMemorySize=2g\"]\r\n    }\r\n  },\r\n  {\r\n    \"type\": \"compact\",\r\n    \"dataSource\": \"sys_storage_log\",\r\n    \"tuningConfig\": {\r\n      \"type\": \"index_parallel\"\r\n    }\r\n  }\r\n]', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2021-07-21 14:13:57', '', 1, 1704330000000, 1704416400000);
+INSERT INTO `xxl_job_info` VALUES (101, 2, '0 0 0 1/7 * ?', 'run_web_sketch_assessment', '2022-01-27 17:40:42', '2022-02-23 13:34:51', 'galaxy', '', 'FAILOVER', 'httpToStoreJobHandler', '[{\r\n \"url\": \"http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/sql/?option=long_term&resultId=1&query=SELECT server_domain AS name, arrayStringConcat(groupUniqArray(decoded_as),\',\') AS protocol, count(*) AS sessions FROM tsg_galaxy_v3.session_record WHERE recv_time >= toStartOfDay(now()- INTERVAL 7 DAY) AND recv_time < toStartOfDay(now()) AND notEmpty(server_domain) GROUP BY name ORDER BY sessions DESC LIMIT 500000;\",\r\n \"method\": \"get\",\r\n \"requestBody\": {},\r\n \"resultKey\": \"data\",\r\n \"store\": \"all\"\r\n}, {\r\n \"url\": \"http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/sql/?option=long_term&resultId=1&query=SELECT server_domain AS name, arrayStringConcat(groupUniqArray(decoded_as),\',\') AS protocol, count(*) AS sessions FROM tsg_galaxy_v3.session_record WHERE recv_time >= toStartOfDay(now()- INTERVAL 7 DAY) AND recv_time < toStartOfDay(now()) AND notEmpty(server_domain) AND empty(fqdn_category_list) GROUP BY name ORDER BY sessions DESC LIMIT 500000;\",\r\n \"method\": \"get\",\r\n \"requestBody\": {},\r\n \"resultKey\": \"data\",\r\n \"store\": \"uncategorized\"\r\n}]', 'COVER_EARLY', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2022-01-27 17:40:42', '', 1, 1704067200000, 1704672000000);
+INSERT INTO `xxl_job_info` VALUES (141, 2, '0 0 2 * * ?', 'set_clickhouse_table_and_field_ttl', '2022-06-07 07:42:38', '2022-06-08 09:28:19', 'wk', '', 'FIRST', 'changeCkTtlJobHandler', '', 'SERIAL_EXECUTION', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2022-06-07 07:42:38', '', 1, 1704333600000, 1704420000000);
+INSERT INTO `xxl_job_info` VALUES (143, 2, '0 0 0/1 * * ?', 'run_ip_learning', '2022-11-09 10:29:20', '2023-12-26 04:01:24', 'galaxy', '', 'FIRST', '', '', 'SERIAL_EXECUTION', 0, 0, 'GLUE_SHELL', '#!/bin/bash\nsh -c \'cd /opt/schedule-service/iplearning/ && ./start.sh\'', '0001', '2023-12-26 04:01:24', '', 1, 1704337200000, 1704340800000);
+INSERT INTO `xxl_job_info` VALUES (145, 2, '0 0 1 * * ?', 'run_dos_baseline', '2022-11-10 06:15:27', '2023-11-13 05:37:41', 'galaxy', '', 'FIRST', '', '', 'SERIAL_EXECUTION', 0, 0, 'GLUE_SHELL', '#!/bin/bash\nsh -c \'cd /opt/schedule-service/dos-baseline && ./start.sh\'', '0000', '2023-11-13 05:37:41', '', 1, 1704330000000, 1704416400000);
+INSERT INTO `xxl_job_info` VALUES (155, 2, '0 0 16 1/7 * ?', 'run_geoip_to_mmdb', '2022-12-02 06:56:18', '2023-12-21 09:44:44', 'galaxy', '', 'FAILOVER', 'ipLocateLibraryJobHandler', '{\r\n  \"command\": \"docker run --rm --network=host  -v {{ deploy_dir }}/schedule-service/mmdb:/mmdb --name perl-mmdb perl:v3.1 sh -c \'cd /mmdb && perl mmdb-perl.pl\' \",\r\n  \"path\": \"/opt/schedule-service/mmdb\",\r\n  \"knowledgeBaseServer\": \"http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/v1/knowledge_base\",\r\n    \"knowledgeBaseList\": [{\"name\": \"ip_builtin\",\"builtin\":1 ,\"kb_id\": \"64af7077-eb9b-4b8f-80cf-2ceebc89bea9\" },\r\n                        {\"name\": \"ip_user_defined\",\"builtin\":0 ,\"kb_id\": \"004390bc-3135-4a6f-a492-3662ecb9e289\" }]\r\n}', 'DISCARD_LATER', 0, 0, 'BEAN', '#!/bin/bash\nsh -c \'cd /opt/schedule-service/dos-baseline && ./start.sh\'', 'GLUE代码初始化', '2022-12-02 06:56:18', '', 0, 0, 0);
+INSERT INTO `xxl_job_info` VALUES (158, 2, '0 0 1 * * ?', 'run_druid_segement_granularity_compact', '2023-02-20 05:59:59', '2023-05-11 03:39:14', 'galaxy', '', 'FAILOVER', 'druidMergeHistoryDataJobHandler', '[\r\n    {\r\n        \"dataSource\": \"service_function_status\",\r\n        \"internalOrNumOfHistory\": \"1\",\r\n        \"queryGranularity\": \"PT5M\",\r\n        \"segmentGranularity\": \"DAY\"\r\n    },\r\n    {\r\n        \"dataSource\": \"service_chaining_rule_hits\",\r\n        \"internalOrNumOfHistory\": \"1\",\r\n        \"queryGranularity\": \"PT5M\",\r\n        \"segmentGranularity\": \"DAY\"\r\n    },\r\n    {\r\n        \"dataSource\": \"traffic_shaping_rule_hits\",\r\n        \"internalOrNumOfHistory\": \"1\",\r\n        \"queryGranularity\": \"PT5M\",\r\n        \"segmentGranularity\": \"DAY\"\r\n    }\r\n]', 'DISCARD_LATER', 0, 0, 'BEAN', '', 'GLUE代码初始化', '2023-02-20 05:59:59', '', 1, 1704330000000, 1704416400000);
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/application.properties.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/application.properties.j2
new file mode 100644
index 0000000..205a6a0
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/application.properties.j2
@@ -0,0 +1,23 @@
+#配置集的配置格式
+nacos.config.type=properties
+#配置中心地址
+nacos.config.server-addr={{ vrrp_instance.default.virtual_ipaddress }}:8848
+#命名空间
+nacos.config.namespace={{ config_namespace }}
+#数据集ID
+nacos.config.data-id=galaxy-job-executor.properties
+#开启自动刷新
+nacos.config.auto-refresh=true
+#配置对应的分组
+nacos.config.group=Galaxy
+#Nacos认证用户
+nacos.config.username=nacos
+#Nacos认证密码
+nacos.config.password=nacos
+#开启配置预加载功能
+nacos.config.bootstrap.enable=true
+#开启Nacos支持日志级别的加载时机
+nacos.config.bootstrap.log-enable=true
+
+### log config
+logging.config=./config/logback.xml
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/docker-compose.yml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/docker-compose.yml.j2
new file mode 100644
index 0000000..1aab54c
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/docker-compose.yml.j2
@@ -0,0 +1,19 @@
+version: '3'
+services:
+  galaxy-job-executor:
+    image: {{ image_name }}:{{ image_tag }}
+    container_name: galaxy-job-executor
+    environment:
+      JAVA_OPTS: "{{ galaxy_job_service.executor_java_opts }}"
+    ports:
+      - "8182:8182"
+    volumes:
+      - "{{ deploy_dir }}/{{ container_name }}/logs:/logs"
+      - "{{ deploy_dir }}/{{ container_name }}/data:/data"
+      - "{{ deploy_dir }}/{{ container_name }}/config:/home/tsg/galaxy/galaxy-job/galaxy-job-executor/config"
+      - "{{ deploy_dir }}/schedule-service:/opt/schedule-service"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+      - "/usr/bin/docker:/usr/bin/docker"
+    restart: always
+    network_mode: "host"
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/galaxy-job-executor.properties.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/galaxy-job-executor.properties.j2
new file mode 100644
index 0000000..5c8b0ef
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/galaxy-job-executor.properties.j2
@@ -0,0 +1,65 @@
+################################静态参数配置(修改后需要重启项目)################################
+### web port
+server.port = 8185
+spring.application.name=galaxy-job-executor
+### actuator
+management.server.servlet.context-path=/actuator
+management.health.mail.enabled=false
+management.endpoints.web.exposure.include=*
+#详细的应用健康信息
+management.endpoint.health.show-details=always
+management.endpoint.metrics.enabled=true
+management.endpoint.prometheus.enabled=true
+management.metrics.export.prometheus.enabled=true
+management.metrics.tags.application=${spring.application.name}
+zookeeper.server={{ zookeeper_servers }}
+
+################################动态参数配置(修改后不需要重启项目)################################
+##存储配额文件服务器
+storge.files.hos-server={{ data_center_name }}|{{ vrrp_instance.oss.virtual_ipaddress }}:9098
+storge.files.token={{ hos_token }}
+##存储配额查询druid
+storge.analytic.server={{ data_center_name }}|{{ vrrp_instance.default.virtual_ipaddress }}:8089
+##存储配额查询clickhouse
+storge.traffic.server={{ data_center_name }}|{{ vrrp_instance.default.virtual_ipaddress }}:8124
+storge.traffic.datasource=tsg_galaxy_v3
+storge.traffic.username=ENC(CNpgJlarVTNItkyW6DAQWA==)
+storge.traffic.password=ENC(LDEb2OekU7iZWiFw6pUYBSozVKP27r1y)
+#删除ttl
+storge.traffic.system.parts=system.parts
+#存储配额查询
+storge.traffic.system.partsclusters=system.parts_cluster
+storge.traffic.system.disks=system.disks_cluster
+storge.traffic.system.tables=system.tables_cluster
+storge.traffic.system.clusters=system.clusters
+#删除ttl白名单,多个逗号分隔
+storge.files.delete.exclusion=default,assessment_hos_bucket,knowledge_base_hos_bucket
+storge.analytic.delete.exclusion=traffic_general_stat
+storge.taffic.delete.exclusion=assessment_event_local
+### xxl-job admin address list, such as "http://address" or "http://address01,http://address02"
+xxl.job.admin.addresses={{ job_admin_servers }}
+### xxl-job, access token
+xxl.job.accessToken=
+### xxl-job executor registry-address: default use address to registry , otherwise use ip:po
+xxl.job.executor.appname=galaxy-executor
+### xxl-job executor registry-address: default use address to registry , otherwise use ip:port if address is null
+xxl.job.executor.address=
+### xxl-job executor server-info
+xxl.job.executor.ip=
+xxl.job.executor.port=8886
+### xxl-job executor log-path
+xxl.job.executor.logpath=/data/logs/jobhandler
+### xxl-job executor log-retention-days
+xxl.job.executor.logretentiondays=30
+## http pool config
+### max connection number
+http.pool.max.connection=500
+http.pool.request.timeout=120000
+http.pool.response.timeout=300000
+http.pool.max.per.route=300
+http.pool.connect.timeout=10000
+##指定kafka server的地址，集群配多个，中间，逗号隔开
+spring.kafka.bootstrap-servers={{ kafka_sink_servers }}
+spring.kafka.ssl.enable=true
+spring.kafka.ssl.username=ENC(RYN0it9o1vYynI1xlmX/Gw==)
+spring.kafka.ssl.pin=ENC(vlb4F5Y3ff0Am/3MMPdIpYjZq8wcGNMd)
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/logback.xml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/logback.xml.j2
new file mode 100644
index 0000000..758d04d
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/executor/logback.xml.j2
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration scan="true" scanPeriod="60 seconds" debug="false">
+
+    <!--每100M压缩日志文件-->
+    <property name="LOG_SIZE" value="100MB"/>
+    <!--日志文件路径-->
+    <property name="LOG_PATH" value="/logs"/>
+    <!--日志文件名称-->
+    <property name="LOG_FILE_NAME" value="galaxy-job-executor"/>
+    <!--日志打印等级-->
+    <property name="LOG_LEVEL" value="error"/>
+    <!--日志最大的历史30天 -->
+    <property name="LOG_DAYS" value="30"/>
+    <!--日志打印格式-->
+    <property name="LOG_PATTERN" value="%d{yyyy-MM-dd HH:mm:ssZ,UTC} [%thread] %-5level %logger -%msg%n"/>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>${LOG_PATTERN}</pattern>
+        </encoder>
+    </appender>
+    <appender name="ALL" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${LOG_PATH}/${LOG_FILE_NAME}.log</file>
+        <filter class="ch.qos.logback.classic.filter.LevelFilter">
+            <level>ALL</level>
+        </filter>
+        <encoder>
+            <pattern>${LOG_PATTERN}</pattern>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>
+                ${LOG_PATH}/history/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz
+            </fileNamePattern>
+            <maxHistory>${LOG_DAYS}</maxHistory>
+            <maxFileSize>${LOG_SIZE}</maxFileSize>
+        </rollingPolicy>
+    </appender>
+
+    <root level="${LOG_LEVEL}">
+        <appender-ref ref="ALL"/>
+        <appender-ref ref="STDOUT"/>
+    </root>
+
+</configuration>
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/push_config.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/push_config.sh.j2
new file mode 100644
index 0000000..87689e5
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/push_config.sh.j2
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+config_path_root=$1
+nacos_server={{ vrrp_instance.default.virtual_ipaddress }}
+
+tenant="{{ config_namespace }}"
+app_name="galaxy-job-service"
+
+shell_result=1
+
+# 推送配置
+function push() {
+  local config_file=$1
+  local data_id="${config_file##*/}"
+  local suffix="${config_file##*.}"
+
+  [ $suffix == "sql" ] && suffix="text"
+  [ $suffix == "yml" ] && suffix="yaml"
+
+  local nacos_push_url="http://$nacos_server:8848/nacos/v1/cs/configs?username=nacos&password=nacos&tenant=$tenant&group=Galaxy"
+  local result=$(curl --retry-delay 3 --retry 3 -sw '%{http_code}' -o /dev/null --request POST "$nacos_push_url&dataId=$data_id&appName=$app_name&type=$suffix" --data-urlencode content@"$config_file")
+  if [[ $result -eq '200' ]]; then
+    sleep 2
+    check=$(curl -sw '%{http_code}' -o /dev/null -X GET "$nacos_push_url&dataId=$data_id")
+    #echo "$check $data_id"
+
+    if [[ $check -ne '200' ]]; then
+      shell_result=0
+    fi
+  fi
+}
+
+# 获取本地文件
+function pushConfig() {
+  for config_path in $(readlink -f $config_path_root); do
+    if [ -d $config_path ]; then
+      for file in $(ls $config_path); do
+        config_path_file=$config_path/$file
+        if [ -f $config_path_file ]; then
+          push $config_path_file
+        fi
+      done
+    fi
+    if [ -f $config_path ]; then
+      push $config_path
+    fi
+  done
+}
+
+pushConfig
+echo $shell_result
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/upgrade-mariadb-data.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/upgrade-mariadb-data.sh.j2
new file mode 100644
index 0000000..bfd7c17
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/templates/upgrade-mariadb-data.sh.j2
@@ -0,0 +1,50 @@
+#!/bin/bash
+set -e
+
+# 数据库连接参数
+upgrade_dir=${1:-`pwd`/upgrade-sql}
+
+# 指定当前版本和历史版本
+current_version="{{ solution_version }}"
+old_version="{{ old_version }}"
+
+# 数据库连接参数
+db_hostname="{{ vrrp_instance.default.virtual_ipaddress }}"
+db_name="xxl_job"
+db_username="root"
+db_password="{{ mariadb_default_pin }}"
+
+suffix=".sql"
+# 获取更新目录中的所有SQL文件
+sql_files=$(find "${upgrade_dir}" -name "*$suffix")
+
+# 标记是否开始执行历史版本的标识
+start_execution=false
+
+# 循环处理每个SQL文件
+for file in ${sql_files}; do
+    # 从文件名中提取版本号
+    filename=$(basename "$file")
+    version=$(echo "$filename" | grep -oE '[0-9]+(\.[0-9]+)?' | tr -d '.')
+    current_version="${current_version//./}"
+    old_version="${old_version//./}"
+    # 检查是否达到历史版本
+    if [[ "${version}" -gt "${old_version}" ]]; then
+        start_execution=true
+    fi
+
+    # 检查是否开始执行历史版本之后的SQL文件
+    if [[ ${start_execution} = true ]]; then
+        echo "Executing SQL file: ${file}"
+
+        # 执行SQL文件
+        mysql -u "${db_username}" -p"${db_password}" -h "${db_hostname}" "${db_name}" < "${file}"
+        echo "Upgrade ${version} successfully"
+    fi
+
+    # 检查是否达到当前版本
+    if [[ "${version}" -ge "${current_version}" ]]; then
+        break
+    fi
+done
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-job-service/vars/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-job-service/vars/main.yml
new file mode 100644
index 0000000..de010b3
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-job-service/vars/main.yml
@@ -0,0 +1,26 @@
+software_packages_dir: "{{ role_path }}/../software-packages"
+
+admin_image_name: galaxy-job-admin
+executor_image_name: galaxy-job-executor
+
+admin_image_tag: v1.3.220308
+executor_image_tag: v1.3.231220
+
+admin_container_name: galaxy-job-admin
+executor_container_name: galaxy-job-executor
+
+
+mariadb_job_database: xxl_job
+job_executor_servers: "{% for host in groups['galaxy_job_service'] %}http://{{ host }}:8886/{% if not loop.last %},{% endif %}{% endfor %}"
+
+job_admin_servers: "{% for host in groups['galaxy_job_service'] %}http://{{ host }}:8184/xxl-job-admin{% if not loop.last %},{% endif %}{% endfor %}"
+
+kafka_sink_servers: "{% for host in groups['kafka'] %}{{ host }}:9094{% if not loop.last %},{% endif %}{% endfor %}"
+zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ host }}:2181{% if not loop.last %},{% endif %}{% endfor %}"
+
+backup_path: "{{ deploy_dir }}/backup/tsg-olap/{{ old_version }}/"
+
+backup_items:
+  - config
+  - docker-compose.yml
+  - init
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/handlers/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/handlers/main.yml
new file mode 100644
index 0000000..e6232c2
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/handlers/main.yml
@@ -0,0 +1,42 @@
+- name: Stop Container
+  docker_container:
+    name: '{{ image_name }}'
+    state: absent
+    force_kill: true
+
+- name: Remove Image
+  docker_image:
+    name: '{{ image_name }}'
+    tag: '{{ image_tag }}'
+    state: absent
+    force_absent: true
+
+- name: Copy Image
+  copy:
+    src: 'files/{{ image_name }}-{{ image_tag }}.tar'
+    dest: '{{ deploy_dir }}/{{ container_name }}'
+    force: yes
+
+- name: Unarchive Image
+  unarchive:
+    src: 'files/{{ image_name }}-{{ image_tag }}.tar.gz'
+    dest: '{{ deploy_dir }}/{{ container_name }}'
+    force: yes 
+
+- name: Load Image
+  docker_image:
+    name: '{{ image_name }}'
+    tag: '{{ image_tag }}'
+    load_path: '{{ deploy_dir }}/{{ container_name }}/{{ image_name }}-{{ image_tag }}.tar'
+    source: load
+    force_tag: yes
+    force_source: yes
+    timeout: 300
+
+- name: Start Container
+  docker_compose:
+    project_src: '{{ deploy_dir }}/{{ container_name }}'
+
+- name: Start Nginx
+  shell: 'docker restart galaxy-gateway-nginx'
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/backup.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/backup.yml
new file mode 100644
index 0000000..899461b
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/backup.yml
@@ -0,0 +1,25 @@
+- name: Check if backup directory exists
+  stat:
+    path: "{{ backup_path }}"
+  register: exist_status
+
+- name: Backup directories
+  block:
+    - name: Creat backup directory
+      file:
+        state: directory
+        path: "{{ backup_path }}"
+
+    - name: Copying directories to backup
+      copy:
+        src: "{{ deploy_dir }}/{{ container_name }}/{{ item }}"
+        dest: "{{ backup_path }}"
+        remote_src: true
+      loop: "{{ backup_items }}"
+
+  rescue:
+    - name: Remove backup directory on failure
+      file:
+        path: "{{ backup_path }}"
+        state: absent
+  when: exist_status.stat.exists == false
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/check.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/check.yml
new file mode 100644
index 0000000..37029eb
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/check.yml
@@ -0,0 +1,24 @@
+- name: Wait for galaxy-qgw-service port 8183 to become available
+  wait_for:
+    host: "{{ inventory_hostname }}"
+    port: 8183
+    state: started
+    timeout: 120
+
+- name: Request component status
+  uri:
+    url: "http://{{ inventory_hostname }}:8183/v1/monitor/component_status"
+    body_format: "json"
+    status_code: [200,503]
+  register: response_health
+
+- name: Checking component status
+  debug:
+    msg: "{{ response_health.json.components | dict2items | json_query(query) }}"
+  vars:
+    query: "[?value.status == 'DOWN']"
+  failed_when: response_health.json.status == "DOWN"
+
+- name: Checking metadata
+  uri:
+    url: "http://{{ inventory_hostname }}:8183/v1/troubleshooting/sanity?test=schema_consistency"
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/deploy.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/deploy.yml
new file mode 100644
index 0000000..90125a3
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/deploy.yml
@@ -0,0 +1,47 @@
+- name: Creating config directory
+  file:
+    state: directory
+    path: "{{ deploy_dir }}/{{ container_name }}/config"
+
+- name: Copying config template
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ deploy_dir }}/{{ container_name }}/{{ item.dest }}"
+    force: yes
+  loop:
+    - {src: 'docker-compose.yml.j2', dest: 'docker-compose.yml'}
+    - {src: 'application.yml.j2', dest: 'config/application.yml'}
+    - {src: 'hazelcast.yaml.j2', dest: 'config/hazelcast.yaml'}
+  notify:
+    - Start Container
+
+- name: Copying config files
+  copy:
+    src: "files/config/"
+    dest: "{{ deploy_dir }}/{{ container_name }}/config/"
+    force: yes
+  notify:
+    - Start Container
+
+- name: Copying image to {{ deploy_dir }}/{{ container_name }}/{{ image_name }}-{{ image_tag }}.tar
+  copy:
+    src: '{{ software_packages_dir }}/{{ image_name }}-{{ image_tag }}.tar'
+    dest: '{{ deploy_dir }}/{{ container_name }}/'
+    force: true
+  notify:
+    - Load Image
+    - Start Container
+
+- name: Copying nginx config and restart nginx
+  template:
+    src: "{{ container_name }}.conf.j2"
+    dest: "{{ deploy_dir }}/galaxy-gateway-nginx/conf/includes/{{ container_name }}.conf"
+    force: yes
+  notify:
+    - Start Nginx
+  delegate_to: "{{ item }}"
+  run_once: true
+  loop: "{{ groups.loadbalancer[:2] }}"
+
+- meta: flush_handlers
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/init.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/init.yml
new file mode 100644
index 0000000..4c677ad
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/init.yml
@@ -0,0 +1,60 @@
+- name: Check if directory exists
+  stat:
+    path: "{{ deploy_dir }}/{{ container_name }}/init"
+  register: directory_stat
+  ignore_errors: true
+
+- name: Remove files and directories
+  file:
+    path: "{{ deploy_dir }}/{{ container_name }}/init"
+    state: absent
+  when: directory_stat.stat.exists
+
+- name: Creating init directory
+  file:
+    state: directory
+    path: "{{ deploy_dir }}/{{ container_name }}/init"
+
+- name: Copying {{ container_name }} init template
+  template:
+    src: "{{ item.src }}.j2"
+    dest: "{{ deploy_dir }}/{{ container_name }}/init/{{ item.src }}"
+    mode: "{{ item.mode }}"
+    force: true
+  with_items:
+    - { src: "push_config.sh", mode: "0755" }
+    - { src: "galaxy-qgw-service.yml", mode: "0644" }
+    - { src: "create_mariadb_tables.sh", mode: "0755" }
+
+- name: Pushing config to nacos
+  shell: 'cd {{ deploy_dir }}/{{ item.src }}/init && sh push_config.sh {{ item.conf }}'
+  register: post_result
+  until: post_result.stdout == '1'
+  retries: 3
+  delay: 3
+  failed_when: post_result.stdout != '1'
+  loop:
+    - { src: "{{ container_name }}" , conf: "galaxy-qgw-service.yml" }
+  run_once: true
+
+- name: Copying init files
+  copy:
+    src: "files/{{ item }}"
+    dest: "{{ deploy_dir }}/{{ container_name }}/init/"
+    force: yes
+  loop:
+    - 'data'
+    - 'schema'
+    - 'schema-upgrade-v3.1.jar'
+
+- name: Create database
+  shell: cd "{{ deploy_dir }}/{{ container_name }}/init"&& source /etc/profile && sh create_mariadb_tables.sh
+  run_once: true
+
+- name: Push schema to nacos
+  shell: cd "{{ deploy_dir }}/{{ container_name }}/init"&& source /etc/profile && java -jar schema-upgrade-v3.1.jar schema/ {{ vrrp_instance.default.virtual_ipaddress }} {{ config_namespace }}
+  run_once: true
+
+- name: Push schema to nacos
+  shell: cd "{{ deploy_dir }}/{{ container_name }}/init"&& source /etc/profile && sh push_config.sh galaxy-qgw-service.yml
+  run_once: true
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/main.yml
new file mode 100644
index 0000000..80fea0f
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/main.yml
@@ -0,0 +1,17 @@
+- block:  
+  - include: uninstall.yml
+  - include: init.yml
+  - include: deploy.yml
+  - include: check.yml
+  when: operation == 'install'
+
+- block:
+  - include: check.yml
+  - include: backup.yml
+  - include: upgrade.yml
+  - include: check.yml
+  when: operation == 'upgrade'
+
+- block: 
+  - include: uninstall.yml
+  when: operation == 'uninstall'
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/uninstall.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/uninstall.yml
new file mode 100644
index 0000000..b623ddd
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/uninstall.yml
@@ -0,0 +1,18 @@
+- name: Stopping and removing {{ container_name }} container
+  docker_container:
+    name: '{{ container_name }}'
+    state: absent
+    force_kill: true
+
+- name: Removing old {{ image_name }} image
+  docker_image:
+    name: '{{ image_name }}'
+    tag: '{{ image_tag }}'
+    state: absent
+    force_absent: true
+
+- name: Ansible delete old {{ deploy_dir }}/{{ container_name }}
+  file:
+    path: '{{ deploy_dir }}/{{ container_name }}'
+    state: absent
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/upgrade.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/upgrade.yml
new file mode 100644
index 0000000..c190537
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/tasks/upgrade.yml
@@ -0,0 +1,95 @@
+- name: Check if directory exists
+  stat:
+    path: "{{ deploy_dir }}/{{ container_name }}/init"
+  register: directory_stat
+  ignore_errors: true
+
+- name: Remove files and directories
+  file:
+    path: "{{ deploy_dir }}/{{ container_name }}/init"
+    state: absent
+  when: directory_stat.stat.exists
+
+- name: Creating init directory
+  file:
+    state: directory
+    path: "{{ deploy_dir }}/{{ container_name }}/init"
+
+- name: Copying {{ container_name }} init template
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ deploy_dir }}/{{ container_name }}/init/{{ item.dest }}"
+    mode: 0755
+    force: true
+  with_items:
+    - { src: "push_config.sh.j2", dest: "push_config.sh" }
+    - { src: "create_mariadb_tables.sh.j2", dest: "create_mariadb_tables.sh" }
+    - { src: "merge-config.sh.j2", dest: "merge-config.sh" }
+    - { src: 'keys.txt', dest: 'keys.txt'}
+
+- name: Copying init files
+  copy:
+    src: "files/{{ item }}"
+    dest: "{{ deploy_dir }}/{{ container_name }}/init/"
+    force: yes
+  loop:
+    - 'data'
+    - 'schema'
+    - 'schema-upgrade-v3.1.jar'
+
+- name: Copying config files
+  copy:
+    src: "files/config/"
+    dest: "{{ deploy_dir }}/{{ container_name }}/config/"
+    force: yes
+  notify:
+    - Start Container
+
+- name: Copying config template
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ deploy_dir }}/{{ container_name }}/{{ item.dest }}"
+    force: yes
+  loop:
+    - {src: 'docker-compose.yml.j2', dest: 'init/docker-compose.yml.j2'}
+    - {src: "galaxy-qgw-service.yml.j2", dest: "init/galaxy-qgw-service.yml.j2"}
+    - {src: 'application.yml.j2', dest: 'config/application.yml'}
+    - {src: 'hazelcast.yaml.j2', dest: 'config/hazelcast.yaml'}
+  notify:
+    - Start Container
+
+- name: Merge online config 
+  shell: cd {{ deploy_dir }}/{{ container_name }}/init && sh merge-config.sh galaxy-qgw-service.yml galaxy-qgw-service.yml.j2 nacos
+
+- name: Merge online docker-compose.yml        
+  shell: cd {{ deploy_dir }}/{{ container_name }}/init && sh merge-config.sh {{ deploy_dir }}/{{ container_name }}/docker-compose.yml docker-compose.yml.j2
+
+- name: Create database
+  shell: cd {{ deploy_dir }}/{{ container_name }}/init && source /etc/profile && sh create_mariadb_tables.sh
+  run_once: true
+
+- name: Push schema to nacos
+  shell: cd {{ deploy_dir }}/{{ container_name }}/init && source /etc/profile && java -jar schema-upgrade-v3.1.jar schema/ {{ vrrp_instance.default.virtual_ipaddress }} {{ config_namespace }}
+  run_once: true
+
+- name: Push config to nacos
+  shell: 'cd {{ deploy_dir }}/{{ item.src }}/init && source /etc/profile && sh push_config.sh {{ item.conf }}'
+  register: post_result
+  until: post_result.stdout == '1'
+  retries: 3
+  delay: 3
+  failed_when: post_result.stdout != '1'
+  loop:
+    - { src: "{{ container_name }}" , conf: "galaxy-qgw-service.yml" }
+  run_once: true
+
+- name: Copying image to {{ deploy_dir }}/{{ container_name }}/{{ image_name }}-{{ image_tag }}.tar
+  copy:
+    src: '{{ software_packages_dir }}/{{ image_name }}-{{ image_tag }}.tar'
+    dest: '{{ deploy_dir }}/{{ container_name }}/'
+    force: true
+  notify:
+    - Load Image
+    - Start Container
+
+- meta: flush_handlers
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/application.yml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/application.yml.j2
new file mode 100644
index 0000000..5c103a5
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/application.yml.j2
@@ -0,0 +1,22 @@
+nacos:
+  config:
+    type: yaml #配置集的配置格式
+    server-addr: {{ vrrp_instance.default.virtual_ipaddress }}:8848 #配置中心地址
+    namespace: {{ config_namespace }} #命名空间
+    data-id: galaxy-qgw-service.yml # 数据集ID
+    auto-refresh: true #开启自动刷新
+    group: Galaxy  #配置对应的分组
+    username: nacos #Nacos认证用户
+    password: nacos
+    bootstrap:
+      enable: true #开启配置预加载功能
+      log:
+        enable: true #开启Nacos支持日志级别的加载时机
+spring:
+  servlet:
+    multipart:
+      max-file-size: 5368709120
+      max-request-size: 5368709120
+logging:
+  config: ./config/log4j2.xml
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/create_mariadb_tables.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/create_mariadb_tables.sh.j2
new file mode 100644
index 0000000..46c72fa
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/create_mariadb_tables.sh.j2
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+mysql -uroot -p{{ mariadb_default_pin }} -P3306 -h{{ inventory_hostname  }} -e "create database if not exists {{ mariadb_common_database }} default character set utf8mb4 collate utf8mb4_general_ci;"
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/docker-compose.yml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/docker-compose.yml.j2
new file mode 100644
index 0000000..765b4c5
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/docker-compose.yml.j2
@@ -0,0 +1,14 @@
+version: "3"
+services:
+  galaxy-qgw-service:
+    image: {{ image_name }}:{{ image_tag }}
+    environment:
+      JAVA_OPTS: "{{ galaxy_qgw_service.java_opts }}"
+    container_name: {{ container_name }}
+    ports:
+      - "8183:8183"
+    volumes:
+      - "{{ deploy_dir }}/{{ container_name }}/logs:/home/tsg/galaxy/galaxy-qgw-service/logs"
+      - "{{ deploy_dir }}/{{ container_name }}/config:/home/tsg/galaxy/galaxy-qgw-service/config"
+    restart: always
+    network_mode: "host"
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/galaxy-qgw-service.conf.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/galaxy-qgw-service.conf.j2
new file mode 100644
index 0000000..9f90ded
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/galaxy-qgw-service.conf.j2
@@ -0,0 +1,24 @@
+upstream qgwService {
+
+{% for host in groups['galaxy_qgw_service'] %}
+    server {{ host }}:8183;
+{% endfor %}
+
+}
+
+server {
+    listen  9999;
+    server_name  localhost;
+
+    location / {
+        proxy_pass http://qgwService; #请求转发到查询引擎集群
+        proxy_http_version 1.1; #指定使用http1.1版本
+        proxy_read_timeout 21600; #等待后端服务响应的最大时长
+        gzip on; #开启压缩
+        gzip_comp_level 6; #压缩级别
+        gzip_min_length 1k; #启用gzip压缩的最小文件,小于设置值的文件将不会压缩
+        gzip_types application/json; #压缩文件类型
+        gzip_vary on; #是否传输gzip压缩标志
+    }
+}
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/galaxy-qgw-service.yml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/galaxy-qgw-service.yml.j2
new file mode 100644
index 0000000..4941b2d
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/galaxy-qgw-service.yml.j2
@@ -0,0 +1,175 @@
+##############静态参数配置(修改后需要重启项目)##############
+## 服务配置
+server:
+  port: 8183
+## 监控参数
+management:
+  metrics:
+    tags:
+      application: ${project.name}
+  endpoint:
+    health:
+      show-details: always
+    shutdown:
+      enabled: true
+  health:
+    redis:
+      enabled: false
+    db:
+      enabled: false
+  endpoints:
+    web:
+      exposure:
+        include: '*'
+        exclude: env,auditevents,beans,conditions,info
+      base-path: /v1/monitor
+      path-mapping:
+        health: /component_status
+#MySql configuration
+spring:
+  sleuth:
+    enabled: true
+    traceId128: true
+  datasource:
+    driver-class-name: com.mysql.cj.jdbc.Driver
+    url: jdbc:mysql://{{ vrrp_instance.default.virtual_ipaddress }}:3306/tsg_olap?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=UTC&failOverReadOnly=false&connectTimeout=10000&socketTimeout=30000
+    username: ENC(63aTpwv2vH0vPikW+3Jjig==)
+    password: ENC(LDEb2OekU7iZWiFw6pUYBSozVKP27r1y)
+  flyway:
+    enabled: true              # 是否开启
+    encoding: UTF-8            # 编码
+    sql-migration-prefix: V    # 脚本文件的前缀，默认为V
+    sql-migration-separator: __ #  双下划线
+    baseline-on-migrate: true  # 连接数据库中存在表时设置为true
+    check-location: false
+    locations: filesystem:./config/flyway   # 脚本路径
+    clean-disabled: true      # flyway 的 clean 命令会删除指定 schema 下的所有 table, 生产务必禁掉。这个默认值是 false 理论上作为默认配置是不科学的
+    validate-on-migrate: true # 执行迁移时是否自动调用验证   当你的 版本不符合逻辑 比如 你先执行了 DML 而没有 对应的DDL 会抛出异常
+    placeholder-replacement: false  # 不做取值替换 默认替换为 ${} ，初始化sql中有sql语句存在freemarker替换，所以禁用此项
+## Hbase configuration
+hbase:
+  zookeeperQuorum: {{ zookeeper_noport }}
+  zookeeperZnodeParent: /hbase
+  rpcTimeout: 60000
+  report:
+    dbname: tsg_galaxy
+    tableName: saved_query_result
+    columnFamily: response
+    columnName: result
+##############动态参数配置(修改后不需要重启项目)##############
+## 项目参数
+project:
+  name: galaxy-qgw-service-nacos
+  description: 统一数据查询网关
+  version: 1.1
+  groupId: com.mesalab
+  artifactId: galaxy-qgw-service
+  basedir:
+  corePackage: com.mesalab.common
+  servicePackage: com.mesalab.qgw
+  author:
+    name: darnell
+    url:
+    email:
+## ClickhHouse configuration
+clickhouse:
+  url: http://{{ vrrp_instance.default.virtual_ipaddress }}:8123
+  dbname: tsg_galaxy_v3
+  enableApproximateOptimizer: true
+  realTimeAccount:
+    username: ENC(hYFEuRJJWd93TZg5VbK/o3OXUmxI/irv)
+    pin: ENC(qUA355VopKSx6kwwwXZwqWWEYSu76Slz)
+    socketTimeOut: 120000
+  longTermAccount:
+    username: ENC(z3gVlaa7Q4IMDqtUgvhwoJzKAkH1Uznc)
+    pin: ENC(LDEb2OekU7iZWiFw6pUYBSozVKP27r1y)
+    socketTimeOut: 21700000
+## Druid configuration
+druid:
+  url: {{ vrrp_instance.default.virtual_ipaddress }}:8089/druid/v2/sql
+  dbname: druid
+  socketTimeOut: 120000
+## xxl-job-admin configuration
+xxl-job-admin:
+  url: http://{{ vrrp_instance.default.virtual_ipaddress }}:8181/xxl-job-admin
+  userName: ENC(v8NKKlLWitI7vGhuGYorRQ==)
+  pin: ENC(xTCafSByYA6SyvhUJ6vrh2VSXuZAVK/O)
+## ArangoDB configuration
+arango:
+  server: http://{{ groups.arangodb[0] }}:8529
+  database: tsg_galaxy_v3
+  username: ENC(63aTpwv2vH0vPikW+3Jjig==)
+  pin: ENC(PBdMaxfC3u+HMzjjij2tyuJWeooSuZNW)
+  jwturl: ${arango.server}/_db/${arango.database}/_open/auth
+  queryurl: ${arango.server}/_db/${arango.database}/_api/cursor
+  maxrows: 10000
+  socketTimeOut: 300000
+## hos cfg
+hos:
+  uri: http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos
+  bucket: knowledge_base_bucket
+  token: {{ hos_token }}
+  multiUploadStartSize: 5368709120
+  multiUploadPartSize: 10485760  
+## Engine、Service configuration
+engine:
+  maxCacheNum: 1048575
+  defaultResultNum: 100000
+  sampleDataPeriod: 1800
+service:
+  entity:
+    activeClientIP: 10000
+    topServerIP:
+      TCPBySession: 100000
+      TCPByUniqClientIP: 100000
+      UDPBySession: 100000
+      UDPByUniqClientIP: 40000
+    topSNI:
+      defaultSize: 30000
+      dataset: 2000000
+    gtpc:
+      defaultSize: 100000
+## knowledge value 格式 name:版本
+knowledge:
+  corn: 0 0 */1 * * ?
+  ipLocation:
+      userDefined: 004390bc-3135-4a6f-a492-3662ecb9e289_latest
+      builtIn: 64af7077-eb9b-4b8f-80cf-2ceebc89bea9_latest
+      asn: f9f6bc91-2142-4673-8249-e097c00fe1ea_latest
+## http pool config
+http:
+  pool:
+    connect:
+      timeout: 30000
+    max:
+      connection: 500
+      per:
+        route: 200
+    request:
+      timeout: 30000
+    response:
+      timeout: 120000
+## job cfg
+job:
+  timeSlicing:
+    enabled: true
+    interval: 1800 #单位: Second
+  longPolling:
+    enabled: true
+  interactive:
+    timeout: 10000
+  response:
+    timeout: 3000
+  execution:
+    timeout: 300000
+  pool:
+    corePoolSize: 20
+    maxPoolSize: 30
+    queueCapacity: 10
+## task cfg
+task:
+  pool:
+    corePoolSize: 8
+    maxPoolSize: 16
+    queueCapacity: 1000
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/hazelcast.yaml.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/hazelcast.yaml.j2
new file mode 100644
index 0000000..5d5e99e
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/hazelcast.yaml.j2
@@ -0,0 +1,16 @@
+hazelcast:
+  cluster-name: qgw-cluster
+  network:
+    join:
+      multicast:
+        enabled: false
+      tcp-ip:
+        enabled: true
+        member-list:
+{% for host in groups['galaxy_qgw_service'] %}
+          - {{ host }}
+{% endfor %}
+  map:
+    adHocCache:
+      time-to-live-seconds: 120
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/keys.txt b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/keys.txt
new file mode 100644
index 0000000..0784ff4
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/keys.txt
@@ -0,0 +1 @@
+services.galaxy-qgw-service.environment.JAVA_OPTS
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/merge-config.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/merge-config.sh.j2
new file mode 100644
index 0000000..aba794c
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/merge-config.sh.j2
@@ -0,0 +1,83 @@
+#!/bin/bash
+set -e
+
+host="{{ vrrp_instance.default.virtual_ipaddress }}"
+tenant="{{ config_namespace }}"
+group_id="Galaxy"
+app_name=""
+username="nacos"
+password="nacos"
+
+online_file=$1
+output_file=$1
+template_file=$2
+
+online_file=$1
+output_file=$1
+template_file=$2
+
+[[ -z $1 || -z $2 ]] && echo "Missing input file !" && exit 1
+
+function format_file {
+	local file="$1"
+	echo "format "$file
+	if [ -f "$file" ]; then
+		cat "$file" | tr -d "\r" >"$file"_new
+		mv "$file"_new $file
+	fi
+}
+
+function download_nacos_config() {
+	local data_id=$online_file
+
+	config_url="http://$host:8848/nacos/v1/cs/configs?dataId=$data_id&group=$group_id&appName=$app_name&username=$username&password=$password&tenant=$tenant"
+	online_file=nacos-"$data_id"
+	curl -s -o $online_file $config_url
+	format_file $online_file
+}
+
+function merge_yml_config() {
+	grep -v '^$' keys.txt | while IFS= read -r key; do
+		# 使用yq命令读取YAML文件中的键值
+		value=$(yq eval ".${key}" "$online_file")
+		if [[ -n $value && "$value" != "null" ]]; then
+
+			yq eval ".${key} = \"${value}\"" -i $template_file
+			echo "upgrade "$(yq eval ".${key}" $template_file)
+		fi
+
+	done
+	rm -r "$online_file"
+	mv $template_file $output_file
+	echo -e "\033[32mmerge success\033[0m"
+
+}
+
+function merge_properties_config() {
+	grep -v '^$' keys.txt | while IFS= read -r key; do
+		value=$(grep -E "^[[:blank:]]*${key}[[:blank:]]*=" "$online_file" | cut -d'=' -f1 --complement)
+		if [[ -n $value && "$value" != "null" ]]; then
+			sed -i "s/^[[:blank:]]*${key}[[:blank:]]*=\(.*\)$/${key}=$value/" $template_file
+			echo "upgrade "$(grep -E "^[[:blank:]]*${key}[[:blank:]]*=" "$template_file")
+		fi
+
+	done
+	rm -r "$online_file"
+	mv $template_file $output_file
+	echo -e "\033[32mmerge success\033[0m"
+
+}
+
+[[ $3 == 'nacos' ]] && download_nacos_config
+
+suffix="${online_file##*.}"
+
+case $suffix in
+yaml | yml)
+	merge_yml_config
+	;;
+properties)
+	merge_properties_config
+	;;
+esac
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/push_config.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/push_config.sh.j2
new file mode 100644
index 0000000..a5ea43f
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/push_config.sh.j2
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+config_path_root=$1
+nacos_server={{ vrrp_instance.default.virtual_ipaddress }}
+
+tenant="{{ config_namespace }}"
+app_name="{{ container_name }}"
+
+shell_result=1
+
+# 推送配置
+function push() {
+  local config_file=$1
+  local data_id="${config_file##*/}"
+  local suffix="${config_file##*.}"
+
+  [ $suffix == "sql" ] && suffix="text"
+  [ $suffix == "yml" ] && suffix="yaml"
+
+  local nacos_push_url="http://$nacos_server:8848/nacos/v1/cs/configs?username=nacos&password=nacos&tenant=$tenant&group=Galaxy"
+  local result=$(curl --retry-delay 3 --retry 3 -sw '%{http_code}' -o /dev/null --request POST "$nacos_push_url&dataId=$data_id&appName=$app_name&type=$suffix" --data-urlencode content@"$config_file")
+  if [[ $result -eq '200' ]]; then
+    sleep 2
+    check=$(curl -sw '%{http_code}' -o /dev/null -X GET "$nacos_push_url&dataId=$data_id")
+    #echo "$check $data_id"
+
+    if [[ $check -ne '200' ]]; then
+      shell_result=0
+    fi
+  fi
+}
+
+# 获取本地文件
+function pushConfig() {
+  for config_path in $(readlink -f $config_path_root); do
+    if [ -d $config_path ]; then
+      for file in $(ls $config_path); do
+        config_path_file=$config_path/$file
+        if [ -f $config_path_file ]; then
+          push $config_path_file
+        fi
+      done
+    fi
+    if [ -f $config_path ]; then
+      push $config_path
+    fi
+  done
+}
+
+pushConfig
+echo $shell_result
+
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/upload_knowlodge.sh.j2 b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/upload_knowlodge.sh.j2
new file mode 100644
index 0000000..b82e0d0
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/templates/upload_knowlodge.sh.j2
@@ -0,0 +1,87 @@
+#!/bin/bash
+
+COMMON_PATH={{ deploy_dir }}/{{ qgw_soft_home_path }}
+
+function push_knowledge_file(){
+FILE_NAME=$1
+FILE_PATH=$2
+TYPE=$3
+FORMAT=$4
+
+curl_result=$(curl -s --retry-delay 3 --retry 3 --location --request POST 'http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/knowledge_base/v1' --form 'file=@"'$FILE_PATH'/'$FILE_NAME'.'$FORMAT'"' --form 'name='$FILE_NAME'' --form 'format='$FORMAT'' --form 'type='$TYPE'' | jq '.status')
+
+if [[ $curl_result -ne '200' ]];then
+  echo 0
+  echo "The $FILE_NAME.$FORMAT upload to the knowledge base failed."
+  exit 1
+fi
+}
+
+function push_knowledge_csv(){
+ID=$1
+NAME=$2
+TYPE=$3
+
+curl_result=$(curl -s --location --request POST 'http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/knowledge_base/v1' \
+--form 'file=@"./data/'$NAME'.csv"' --form 'name='$NAME'' --form 'format=csv' --form 'type='$TYPE'' --form 'id='$ID'' | jq '.status')
+
+if [[ $curl_result -ne '200' ]];then
+  echo 0
+  echo "The $FILE_NAME.$FORMAT upload to the knowledge base failed."
+  exit 1
+fi
+}
+
+function push_knowledge_mmdb(){
+ID=$1
+NAME=$2
+TYPE=$3
+
+curl_result=$(curl -s --location --request POST 'http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/knowledge_base/v1' \
+--form 'file=@"./data/'$TYPE'.mmdb"' --form 'name='$NAME'' --form 'format=mmdb' --form 'type='$TYPE'' --form 'id='$ID'' | jq '.status')
+
+if [[ $curl_result -ne '200' ]];then
+  echo 0
+  echo "The $FILE_NAME.$FORMAT upload to the knowledge base failed."
+  exit 1
+fi
+}
+
+function pushConfig(){
+
+{% if project == "tsg" %}
+push_knowledge_file asn_v4 $COMMON_PATH/data asn mmdb
+push_knowledge_file asn_v6 $COMMON_PATH/data asn mmdb
+push_knowledge_file ip_v4_built_in $COMMON_PATH/data ip_location mmdb
+push_knowledge_file ip_v4_user_defined $COMMON_PATH/data ip_location mmdb
+push_knowledge_file ip_v6_built_in $COMMON_PATH/data ip_location mmdb
+push_knowledge_file ip_v6_user_defined $COMMON_PATH/data ip_location mmdb
+{% endif %}
+
+
+{% if project == "cn" %}
+push_knowledge_csv 3 cn_dns_server_info_built_in cn_dns_server_info_built_in
+push_knowledge_csv 4 cn_fqdn_icp_built_in cn_fqdn_icp_built_in
+push_knowledge_csv 5 cn_fqdn_category_built_in cn_fqdn_category_built_in
+push_knowledge_csv 6 cn_fqdn_who_is_built_in cn_fqdn_who_is_built_in
+push_knowledge_csv 7 cn_ioc_malware cn_ioc_malware
+push_knowledge_csv 8 cn_ioc_darkweb cn_ioc_darkweb
+push_knowledge_csv 9 cn_app_category_built_in cn_app_category_built_in
+push_knowledge_csv 10 cn_psiphon3_ip cn_psiphon3_ip
+push_knowledge_csv 11 cn_idc_renter_built_in cn_idc_renter_built_in
+push_knowledge_csv 12 cn_internal_ip_built_in cn_internal_ip_built_in
+push_knowledge_csv 13 cn_link_direction_built_in cn_link_direction_built_in
+
+
+push_knowledge_mmdb 16266e3e2aff061d cn_ip_location_built_in cn_ipv4_location_built_in
+push_knowledge_mmdb f8e1f4021a069b7d cn_ip_location_built_in cn_ipv6_location_built_in
+push_knowledge_mmdb 15a614c6f12f1ad5 cn_ip_asn_built_in cn_ipv4_asn_built_in
+push_knowledge_mmdb f4b2ed071a57b35c cn_ip_asn_built_in cn_ipv6_asn_built_in
+
+{% endif %}
+
+}
+
+pushConfig
+
+echo 1
diff --git a/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/vars/main.yml b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/vars/main.yml
new file mode 100644
index 0000000..1e9b201
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/galaxy-qgw-service/vars/main.yml
@@ -0,0 +1,16 @@
+software_packages_dir: "{{ role_path }}/../software-packages"
+
+image_name: galaxy-qgw-service
+image_tag: 374
+container_name: galaxy-qgw-service
+
+backup_path: "{{ deploy_dir }}/backup/tsg-olap/{{ old_version }}/{{ container_name }}"
+
+backup_items:
+  - config
+  - docker-compose.yml
+  - init
+
+
+mariadb_common_database: tsg_olap
+zookeeper_noport: "{% for host in groups['zookeeper'] %}{{ host }}{% if not loop.last %},{% endif %}{% endfor %}:2181"
diff --git a/tsg-olap/parcels/roles/initialization/hbase/tasks/init.yml b/tsg-olap/parcels/roles/initialization/hbase/tasks/init.yml
new file mode 100644
index 0000000..e016d42
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/hbase/tasks/init.yml
@@ -0,0 +1,54 @@
+- name: Setting Project variable
+  shell: cat /etc/hostname
+  register: hostname_master_out
+  run_once: true
+  delegate_to: "{{ groups.hbase[0] }}"
+
+- name: Setting Project variable
+  shell: cat /etc/hostname
+  register: hostname_slave_out
+  run_once: true
+  delegate_to: "{{ groups.hbase[1] if groups['hbase']|length > 1 else groups.hbase[0] }}"
+
+- name: project to ansible variable
+  set_fact: 
+    hostname_master: "{{hostname_master_out.stdout}}"
+    hostname_slave: "{{hostname_slave_out.stdout}}"
+    init_path: "{{ deploy_dir }}/hbase/init"
+    container_name: 'HMaster'
+
+- name: Create {{ init_path }}
+  file:
+    path: '{{ init_path }}'
+    state: directory
+  
+- name: Copying HBase initshell files to {{ init_path }}
+  template:
+    src: '{{ item }}'
+    dest: '{{ init_path }}/{{ item }}'
+    mode: 0755
+    force: true
+  loop:
+    - create-hbase-table.sh    
+    - create-phoenix-table.sh  
+    - rsgroup.sh
+
+- name: Creating hbase table
+  shell: cd {{ init_path }} && sh create-hbase-table.sh | grep ERROR | grep -v "already exists" 
+  register: result
+  failed_when: "'ERROR' in result.stdout"
+  run_once: true
+
+- name: Creating phoenix table
+  shell: cd {{ init_path }} && sh create-phoenix-table.sh | grep ERROR | grep -v "already exists" 
+  register: result
+  failed_when: "'ERROR' in result.stdout"
+  run_once: true
+
+- name: Enable RsGroup
+  shell: cd {{ init_path }} && sh rsgroup.sh | grep ERROR | egrep -v "already exists|Target RSGroup important is same as source|Source RSGroup important is same as target" 
+  register: result
+  failed_when: "'ERROR' in result.stdout"
+  when: groups.hbase | length > 1 
+  run_once: true
+
diff --git a/tsg-olap/parcels/roles/initialization/hbase/tasks/main.yml b/tsg-olap/parcels/roles/initialization/hbase/tasks/main.yml
new file mode 100644
index 0000000..125d374
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/hbase/tasks/main.yml
@@ -0,0 +1,5 @@
+- include: init.yml
+  when: operation == 'install'
+
+- include: upgrade.yml
+  when: operation == 'upgrade'
diff --git a/tsg-olap/parcels/roles/initialization/hbase/tasks/upgrade.yml b/tsg-olap/parcels/roles/initialization/hbase/tasks/upgrade.yml
new file mode 100644
index 0000000..3b2304b
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/hbase/tasks/upgrade.yml
@@ -0,0 +1,41 @@
+- block:
+  - name: project to ansible variable
+    set_fact:
+      upgrade_path: "{{ deploy_dir }}/hbase/upgrade"
+      container_name: 'HMaster'
+  
+  - name: Clean {{ upgrade_path }}
+    file:
+      path: '{{ upgrade_path }}'
+      state: absent
+  
+  - name: Create {{ upgrade_path }}
+    file:
+      path: '{{ upgrade_path }}'
+      state: directory
+
+  - name: Copying Upgrade files to {{ upgrade_path }}
+    copy:
+      src: "files/upgrade/"
+      dest: "{{ upgrade_path }}/"
+      force: true
+
+  - name: Copying HBase upgrade template to {{ upgrade_path }}
+    template:
+      src: '{{ item }}'
+      dest: '{{ upgrade_path }}/{{ item }}'
+      mode: 0755
+      force: true
+    loop:
+      - upgrade-hbase-data.sh
+      - upgrade-phoenix-data.sh
+  
+  - name: Upgrade hbase table data
+    shell: cd {{ upgrade_path }} && sh upgrade-hbase-data.sh 
+    run_once: true
+  
+  - name: Upgrade phoenix table data
+    shell: cd {{ upgrade_path }} && sh upgrade-phoenix-data.sh
+    run_once: true
+
+  when: inventory_hostname in groups['hbase'][:2]
diff --git a/tsg-olap/parcels/roles/initialization/hbase/templates/create-hbase-table.sh b/tsg-olap/parcels/roles/initialization/hbase/templates/create-hbase-table.sh
new file mode 100644
index 0000000..6dfe60f
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/hbase/templates/create-hbase-table.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+source /etc/profile
+
+docker exec -i {{ container_name }} hbase shell <<EOF
+
+create_namespace 'dos'
+create_namespace 'tsg_galaxy'
+
+create 'tsg_galaxy:saved_query_result', {NAME => 'response', VERSIONS => 1,COMPRESSION => 'GZ',IS_MOB => true, MOB_THRESHOLD => 0}, {NAME => 'detail',COMPRESSION => 'GZ',VERSIONS => 1}
+
+create 'dos:ddos_traffic_baselines', 'TCP SYN Flood','UDP Flood','ICMP Flood','DNS Flood'
+
+
+EOF
diff --git a/tsg-olap/parcels/roles/initialization/hbase/templates/create-phoenix-table.sh b/tsg-olap/parcels/roles/initialization/hbase/templates/create-phoenix-table.sh
new file mode 100644
index 0000000..2fe23d5
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/hbase/templates/create-phoenix-table.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+source /etc/profile
+
+phoenix_path={{ deploy_dir }}/hbase/phoenix-hbase-2.2-5.1.2-bin/bin
+
+$phoenix_path/sqlline.py<<EOF
+
+CREATE schema IF NOT EXISTS "tsg_galaxy";
+
+CREATE view IF NOT EXISTS "tsg_galaxy"."saved_query_result"( ROWKEY VARCHAR PRIMARY KEY, "detail"."excute_sql" VARCHAR, "detail"."read_rows" UNSIGNED_LONG, "detail"."result_id" UNSIGNED_INT, "response"."result" VARCHAR);
+
+
+!quit
+
+EOF
+
diff --git a/tsg-olap/parcels/roles/initialization/hbase/templates/rsgroup.sh b/tsg-olap/parcels/roles/initialization/hbase/templates/rsgroup.sh
new file mode 100644
index 0000000..9e27b1d
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/hbase/templates/rsgroup.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+source /etc/profile
+
+docker exec -i {{ container_name }} hbase shell <<EOF
+
+add_rsgroup 'important'
+
+move_servers_rsgroup 'important',['{{ hostname_master }}:16020']
+
+move_servers_rsgroup 'important',['{{ hostname_slave }}:16020']
+
+flush 'tsg_galaxy:saved_query_result'
+
+move_tables_rsgroup 'important',['tsg_galaxy:saved_query_result']
+
+EOF
+
diff --git a/tsg-olap/parcels/roles/initialization/hbase/templates/upgrade-hbase-data.sh b/tsg-olap/parcels/roles/initialization/hbase/templates/upgrade-hbase-data.sh
new file mode 100644
index 0000000..63245a4
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/hbase/templates/upgrade-hbase-data.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+source /etc/profile
+set -e
+
+# 数据库连接参数
+upgrade_dir=${1:-$(pwd)/upgrade-hbase}
+
+# 指定当前版本和历史版本
+current_version="{{ solution_version }}"
+old_version="{{ old_version }}"
+
+# 数据库连接参数
+container_name="{{ container_name }}"
+
+suffix=".sql"
+# 获取更新目录中的所有SQL文件
+sql_files=$(find "${upgrade_dir}" -name "*$suffix")
+
+# 标记是否开始执行历史版本的标识
+start_execution=false
+
+# 循环处理每个SQL文件
+for file in ${sql_files}; do
+    # 从文件名中提取版本号
+    filename=$(basename "$file")
+    version=$(echo "$filename" | grep -oE '[0-9]+(\.[0-9]+)?' | tr -d '.')
+    current_version="${current_version//./}"
+    old_version="${old_version//./}"
+    # 检查是否达到历史版本
+    if [[ "${version}" -gt "${old_version}" ]]; then
+        start_execution=true
+    fi
+
+    # 检查是否开始执行历史版本之后的SQL文件
+    if [[ ${start_execution} = true ]]; then
+        echo "Executing SQL file: ${file}"
+
+        # 执行SQL文件
+        docker exec -i $container_name hbase shell <"${file}"
+        echo "Upgrade ${version} successfully"
+    fi
+
+    # 检查是否达到当前版本
+    if [[ "${version}" -ge "${current_version}" ]]; then
+        break
+    fi
+done
+
diff --git a/tsg-olap/parcels/roles/initialization/hbase/templates/upgrade-phoenix-data.sh b/tsg-olap/parcels/roles/initialization/hbase/templates/upgrade-phoenix-data.sh
new file mode 100644
index 0000000..da915bc
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/hbase/templates/upgrade-phoenix-data.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+source /etc/profile
+set -e
+
+# 数据库连接参数
+upgrade_dir=${1:-$(pwd)/upgrade-phoenix}
+
+# 指定当前版本和历史版本
+current_version="{{ solution_version }}"
+old_version="{{ old_version }}"
+
+# 数据库连接参数
+phoenix_path={{ deploy_dir }}/hbase/phoenix-hbase-2.2-5.1.2-bin/bin
+
+suffix=".sql"
+# 获取更新目录中的所有SQL文件
+sql_files=$(find "${upgrade_dir}" -name "*$suffix")
+
+# 标记是否开始执行历史版本的标识
+start_execution=false
+
+# 循环处理每个SQL文件
+for file in ${sql_files}; do
+    # 从文件名中提取版本号
+    filename=$(basename "$file")
+    version=$(echo "$filename" | grep -oE '[0-9]+(\.[0-9]+)?' | tr -d '.')
+    current_version="${current_version//./}"
+    old_version="${old_version//./}"
+    # 检查是否达到历史版本
+    if [[ "${version}" -gt "${old_version}" ]]; then
+        start_execution=true
+    fi
+
+    # 检查是否开始执行历史版本之后的SQL文件
+    if [[ ${start_execution} = true ]]; then
+        echo "Executing SQL file: ${file}"
+
+        # 执行SQL文件
+        $phoenix_path/sqlline.py <"$file"
+        echo "Upgrade ${version} successfully"
+    fi
+
+    # 检查是否达到当前版本
+    if [[ "${version}" -ge "${current_version}" ]]; then
+        break
+    fi
+done
+
diff --git a/tsg-olap/parcels/roles/initialization/kafka/tasks/init.yml b/tsg-olap/parcels/roles/initialization/kafka/tasks/init.yml
new file mode 100644
index 0000000..95f7911
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/kafka/tasks/init.yml
@@ -0,0 +1,19 @@
+- name: Set kafka init path
+  set_fact:
+    init_path: '{{ deploy_dir }}/kafka/init'
+
+- name: Create kafka init path
+  file:
+    state: directory
+    path: "{{ init_path }}"
+  
+- name: Copying create_topic.sh to {{ init_path }}
+  template: 
+    src: 'create_topic.sh.j2'
+    dest: '{{ init_path }}/create_topic.sh'
+    mode: 0755
+
+- name: Creating kafka topic
+  shell:  source /etc/profile && cd {{ init_path }} && sh create_topic.sh
+  run_once: true
+
diff --git a/tsg-olap/parcels/roles/initialization/kafka/tasks/main.yml b/tsg-olap/parcels/roles/initialization/kafka/tasks/main.yml
new file mode 100644
index 0000000..05bab68
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/kafka/tasks/main.yml
@@ -0,0 +1,2 @@
+- include: init.yml
+  when: inventory_hostname in groups['kafka'][:2]
diff --git a/tsg-olap/parcels/roles/initialization/kafka/templates/create_topic.sh.j2 b/tsg-olap/parcels/roles/initialization/kafka/templates/create_topic.sh.j2
new file mode 100644
index 0000000..7c61277
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/kafka/templates/create_topic.sh.j2
@@ -0,0 +1,71 @@
+#!/bin/bash
+source /etc/profile
+
+partion_nums={{ groups.kafka | length }}
+replication_nums={{ 2 if groups.kafka | length >1 else 1 }}
+kafka_servers="{% for host in groups['kafka'] %}{{ host }}:9092{% if not loop.last %},{% endif %}{% endfor %}"
+
+common_topic=(
+  BGP-RECORD
+  DOS-EVENT
+  DOS-SKETCH-RECORD
+  NETWORK-TRAFFIC-METRIC
+  OBJECT-STATISTICS-METRIC
+  POLICY-RULE-METRIC
+  PROXY-EVENT
+  PROXY-EVENT-PROCESSED
+  PXY-EXCH-INTERMEDIA-CERT
+  SESSION-RECORD-IPFIX
+  STATISTICS-RULE-METRIC
+  SYS-STORAGE
+  TRAFFIC-FILE-STREAM-RECORD
+  TRAFFIC-TOP-DESTINATION-IP-METRIC
+  TRAFFIC-TOP-METRIC
+  TROUBLESHOOTING-FILE-STREAM-RECORD
+  VOIP-CONVERSATION-RECORD
+  VOIP-RECORD
+  VOIP-RECORD-PROCESSED
+)
+
+multi_partion_topic=(
+  TRANSACTION-RECORD
+  TRANSACTION-RECORD-PROCESSED
+  SESSION-RECORD
+  SESSION-RECORD-PROCESSED
+)
+
+function getTopic() {
+  docker exec kafka kafka-topics.sh --bootstrap-server $kafka_servers --list
+}
+
+function createTopic() {
+  exist_topic=$(echo "$exist_topic_list" | grep -wx "$3" | wc -l)
+  if [ $exist_topic -eq '0' ]; then
+    docker exec kafka kafka-topics.sh '--create' '--bootstrap-server' "$kafka_servers" '--replication-factor' $1 '--partitions' $2 '--topic' $3
+  fi
+}
+
+function checkTopic() {
+  exist_topic_list=$(getTopic)
+  all_topic=("${common_topic[@]}" "${multi_partion_topic[@]}")
+  for topic_name in ${all_topic[@]}; do
+    exist_topic=$(echo "$exist_topic_list" | grep -wx "$topic_name" | wc -l)
+    if [ $exist_topic -eq '0' ]; then
+      echo "Topic $topic_name creation failure"
+      exit 1
+    fi
+  done
+
+}
+
+exist_topic_list=$(getTopic)
+for topic_name in ${common_topic[@]}; do
+  createTopic $replication_nums $partion_nums $topic_name
+done
+
+for topic_name in ${multi_partion_topic[@]}; do
+  createTopic 1 $((partion_nums * 3)) $topic_name
+done
+
+checkTopic
+
diff --git a/tsg-olap/parcels/roles/initialization/kafka/vars/main.yml b/tsg-olap/parcels/roles/initialization/kafka/vars/main.yml
new file mode 100644
index 0000000..d9b473f
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/kafka/vars/main.yml
@@ -0,0 +1 @@
+kafka_servers: "{% for host in groups['kafka'] %}{{ host }}:9092{% if not loop.last %},{% endif %}{% endfor %}"
diff --git a/tsg-olap/parcels/roles/initialization/nacos/tasks/main.yml b/tsg-olap/parcels/roles/initialization/nacos/tasks/main.yml
new file mode 100644
index 0000000..1131a78
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/nacos/tasks/main.yml
@@ -0,0 +1,14 @@
+- name: Create Nacos Namespace
+  uri:
+    url: "http://{{ vrrp_instance.default.virtual_ipaddress }}:8848/nacos/v1/console/namespaces?username=nacos&password={{ nacos_default_pin }}"
+    method: POST
+    body_format: form-urlencoded
+    body:
+      namespaceId: ''
+      namespaceName: "{{ config_namespace }}"
+      namespaceDesc: "{{ config_namespace }}"
+      customNamespaceId: "{{ config_namespace }}"
+  retries: 3
+  delay: 3
+  run_once: true
+  when: operation == 'install'
diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/handlers/main.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/handlers/main.yml
new file mode 100644
index 0000000..5787af9
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/handlers/main.yml
@@ -0,0 +1,43 @@
+- name: Stop Container
+  docker_container:
+    name: '{{ image_name }}'
+    state: absent
+    force_kill: true
+
+- name: Remove Image
+  docker_image:
+    name: '{{ image_name }}'
+    tag: '{{ image_tag }}'
+    state: absent
+    force_absent: true
+
+- name: Copy Image
+  copy:
+    src: 'files/{{ image_name }}-{{ image_tag }}.tar'
+    dest: '{{ deploy_dir }}/{{ container_name }}'
+    force: yes
+
+- name: Unarchive Image
+  unarchive:
+    src: 'files/{{ image_name }}-{{ image_tag }}.tar.gz'
+    dest: '{{ deploy_dir }}/{{ container_name }}'
+    force: yes
+
+- name: Load Image
+  docker_image:
+    name: '{{ image_name }}'
+    tag: '{{ image_tag }}'
+    load_path: '{{ deploy_dir }}/{{ container_name }}/{{ image_name }}-{{ image_tag }}.tar'
+    source: load
+    force_tag: yes
+    force_source: yes
+    timeout: 300
+
+- name: Start Container
+  docker_compose:
+    project_src: '{{ deploy_dir }}/{{ container_name }}'
+
+- name: Start Nginx
+  docker_container:
+    name: 'galaxy-gateway-nginx'
+    state: started
diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/backup.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/backup.yml
new file mode 100644
index 0000000..899461b
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/backup.yml
@@ -0,0 +1,25 @@
+- name: Check if backup directory exists
+  stat:
+    path: "{{ backup_path }}"
+  register: exist_status
+
+- name: Backup directories
+  block:
+    - name: Creat backup directory
+      file:
+        state: directory
+        path: "{{ backup_path }}"
+
+    - name: Copying directories to backup
+      copy:
+        src: "{{ deploy_dir }}/{{ container_name }}/{{ item }}"
+        dest: "{{ backup_path }}"
+        remote_src: true
+      loop: "{{ backup_items }}"
+
+  rescue:
+    - name: Remove backup directory on failure
+      file:
+        path: "{{ backup_path }}"
+        state: absent
+  when: exist_status.stat.exists == false
diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/check.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/check.yml
new file mode 100644
index 0000000..a920d26
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/check.yml
@@ -0,0 +1,23 @@
+- name: Wait for {{ container_name }} port 9093 to become available
+  wait_for:
+    host: "{{ inventory_hostname }}"
+    port: 9093
+    state: started
+    timeout: 120
+
+- name: Request component status
+  uri:
+    url: "http://{{ inventory_hostname }}:9093/monitor"
+    body_format: "json"
+    status_code: [200]
+    return_content: true
+  register: response_health
+
+- name: parse json 
+  set_fact:
+    content_json: "{{ response_health.content | from_json }}"
+
+- name: Checking status message
+  debug:
+    msg: "{{ response_health.content }}"
+  failed_when: content_json.status != "active" and content_json.status != "standby"
diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/deploy.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/deploy.yml
new file mode 100644
index 0000000..c253e7e
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/deploy.yml
@@ -0,0 +1,29 @@
+- name: Creating directory
+  file:
+    state: directory
+    path: '{{ deploy_dir }}/{{ container_name }}/config'
+
+- name: Copying config
+  template:
+    src: "{{ item.src }}"
+    dest: '{{ deploy_dir }}/{{ container_name }}/{{ item.dest }}'
+    force: yes
+  loop: 
+    - { src: 'application.yml.j2', dest: 'config/application.yml' }
+    - { src: 'log4j2-dev.xml.j2', dest: 'config/log4j2-dev.xml' }
+    - { src: 'docker-compose.yml.j2', dest: 'docker-compose.yml' }
+  notify:
+    - Start Container
+
+- name: Copying image to {{ deploy_dir }}/{{ container_name }}
+  copy:
+    src: '{{ software_packages_dir }}/{{ image_name }}-{{ image_tag }}.tar'
+    dest: '{{ deploy_dir }}/{{ container_name }}/'
+    force: true
+  notify:
+    - Load Image
+    - Start Container
+
+
+- meta: flush_handlers
+
diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/init.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/init.yml
new file mode 100644
index 0000000..530e18e
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/init.yml
@@ -0,0 +1,25 @@
+- name: Creating directory
+  file:
+    state: directory
+    path: '{{ deploy_dir }}/{{ container_name }}/init'
+
+- name: Copying init files
+  template:
+    src: "{{ item.src }}.j2"
+    dest: "{{ deploy_dir }}/{{ container_name }}/init/{{ item.src }}"
+    mode: "{{ item.mode }}"
+    force: true
+  loop:
+    - { src: "push_config.sh", mode: "0755" }
+    - { src: "saved-query-scheduler.yml", mode: "0644" }  
+
+- name: Pushing config to nacos
+  shell: 'cd {{ deploy_dir }}/{{ item.src }}/init && sh push_config.sh {{ item.conf }}'
+  register: post_result
+  until: post_result.stdout == '1'
+  retries: 3
+  delay: 3
+  failed_when: post_result.stdout != '1'
+  loop: 
+    - { src: "{{ container_name }}" , conf: "saved-query-scheduler.yml" }
+  run_once: true
diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/main.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/main.yml
new file mode 100644
index 0000000..7f03439
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/main.yml
@@ -0,0 +1,19 @@
+- block:
+  - include: uninstall.yml
+  - include: init.yml
+  - include: deploy.yml
+  - include: check.yml
+  when: operation == 'install'
+
+- block:
+  - include: check.yml
+  - include: backup.yml
+  - include: upgrade.yml
+  - include: init.yml
+  - include: deploy.yml
+  - include: check.yml
+  when: operation == 'upgrade'
+
+- block:
+  - include: uninstall.yml
+  when: operation == 'uninstall'
diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/uninstall.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/uninstall.yml
new file mode 100644
index 0000000..b623ddd
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/uninstall.yml
@@ -0,0 +1,18 @@
+- name: Stopping and removing {{ container_name }} container
+  docker_container:
+    name: '{{ container_name }}'
+    state: absent
+    force_kill: true
+
+- name: Removing old {{ image_name }} image
+  docker_image:
+    name: '{{ image_name }}'
+    tag: '{{ image_tag }}'
+    state: absent
+    force_absent: true
+
+- name: Ansible delete old {{ deploy_dir }}/{{ container_name }}
+  file:
+    path: '{{ deploy_dir }}/{{ container_name }}'
+    state: absent
+
diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/upgrade.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/upgrade.yml
new file mode 100644
index 0000000..c5cb6f2
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/tasks/upgrade.yml
@@ -0,0 +1,22 @@
+- name: Creating config tmp directory
+  file:
+    state: directory
+    path: "/tmp/{{ container_name }}"
+
+- name: Fetching config files
+  fetch:
+    src: "{{ deploy_dir }}/{{ container_name }}/{{ item}}"
+    dest: "/tmp/{{ container_name }}/{{ item}}"
+    flat: yes
+  loop:
+    - 'docker-compose.yml'
+
+- name: Loading variable by docker-compose.yml file
+  include_vars:
+    file: "/tmp/{{ container_name }}/docker-compose.yml"
+    name: docker_config
+
+- name: Set variable
+  set_fact:
+    saved_query_scheduler:
+      java_opts: "{{ docker_config['services']['{{ container_name }}']['environment']['JAVA_OPTS'] }}"
diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/application.yml.j2 b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/application.yml.j2
new file mode 100644
index 0000000..607499b
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/application.yml.j2
@@ -0,0 +1,20 @@
+nacos:
+  config:
+    type: yaml #配置集的配置格式
+    server-addr: {{ vrrp_instance.default.virtual_ipaddress }}:8848 #配置中心地址
+    namespace: {{ config_namespace }} #命名空间
+    data-id: saved-query-scheduler.yml # 数据集ID
+    auto-refresh: true #开启自动刷新
+    group: Galaxy  #配置对应的分组
+    username: nacos #Nacos认证用户
+    password: nacos #Nacos认证密码
+    bootstrap:
+      enable: true #开启配置预加载功能
+      log:
+        enable: true #开启Nacos支持日志级别的加载时机
+spring:
+  profiles:
+    active: dev
+logging:
+  config: ./config/log4j2-dev.xml
+
diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/docker-compose.yml.j2 b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/docker-compose.yml.j2
new file mode 100644
index 0000000..7f8c0e3
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/docker-compose.yml.j2
@@ -0,0 +1,14 @@
+version: "3"
+services:
+  saved-query-scheduler:
+    image: {{ image_name }}:{{ image_tag }}
+    container_name: {{ container_name }}
+    environment:
+      JAVA_OPTS: "{{ saved_query_scheduler.java_opts }}"
+    ports:
+      - "9093:9093"
+    volumes:
+      - "{{ deploy_dir }}/{{ container_name }}/config:/home/tsg/galaxy/saved-query-scheduler/config"
+      - "{{ deploy_dir }}/{{ container_name }}/logs:/home/tsg/galaxy/saved-query-scheduler/logs"
+    restart: always
+    network_mode: 'host'
diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/log4j2-dev.xml.j2 b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/log4j2-dev.xml.j2
new file mode 100644
index 0000000..0e8de70
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/log4j2-dev.xml.j2
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+
+    <!--日志打印相关参数配置-->
+    <Properties>
+        <!--每5M压缩日志文件-->
+        <property name="LOG_SIZE">10M</property>
+        <!--最多产生10个压缩文件-->
+        <property name="LOG_NUMS">10</property>
+        <!--日志打印等级-->
+        <property name="LOG_LEVEL">error</property>
+        <!--日志文件路径-->
+        <property name="LOG_PATH">logs</property>
+        <!--日志文件名称-->
+        <property name="LOG_FILE_NAME">saved-query-scheduler</property>
+        <!--日志打印格式-->
+        <property name="LOG_PATTERN">[%d{yyyy-MM-dd HH:mm:ssZ}{UTC}] [%p] [Thread:%t] %l %x - %m%n</property>
+    </Properties>
+
+    <appenders>
+        <Console name="consoleSystemOutAppender" target="SYSTEM_OUT">
+            <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+            <PatternLayout pattern="${LOG_PATTERN}"/>
+        </Console>
+   
+        <RollingFile name="rollingFileAllAppender"
+                     fileName="${LOG_PATH}/${LOG_FILE_NAME}.log"
+                     filePattern="${LOG_PATH}/history/$${date:yyyy-MM-dd}/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz">
+            <PatternLayout pattern="${LOG_PATTERN}"/>
+            <Policies>
+                <SizeBasedTriggeringPolicy size="${LOG_SIZE}"/>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
+            </Policies>
+            <Filters>
+                <ThresholdFilter level="all" onMatch="ACCEPT" onMismatch="DENY"/>
+            </Filters>
+            <DefaultRolloverStrategy max="${LOG_NUMS}">
+                <Delete basePath="${LOG_PATH}/history" maxDepth="1">
+                    <IfFileName glob="*.log.gz">
+                        <IfLastModified age="30d">
+                            <IfAny>
+                                <IfAccumulatedFileSize exceeds="3 GB" />
+                            </IfAny>
+                        </IfLastModified>
+                    </IfFileName>
+                </Delete>
+            </DefaultRolloverStrategy>
+        </RollingFile>
+    </appenders>
+    <loggers>
+        <root level="${LOG_LEVEL}">
+            <appender-ref ref="consoleSystemOutAppender"/>
+            <appender-ref ref="rollingFileAllAppender"/>
+        </root>
+    </loggers>
+</configuration>
diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/push_config.sh.j2 b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/push_config.sh.j2
new file mode 100644
index 0000000..a5ea43f
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/push_config.sh.j2
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+config_path_root=$1
+nacos_server={{ vrrp_instance.default.virtual_ipaddress }}
+
+tenant="{{ config_namespace }}"
+app_name="{{ container_name }}"
+
+shell_result=1
+
+# 推送配置
+function push() {
+  local config_file=$1
+  local data_id="${config_file##*/}"
+  local suffix="${config_file##*.}"
+
+  [ $suffix == "sql" ] && suffix="text"
+  [ $suffix == "yml" ] && suffix="yaml"
+
+  local nacos_push_url="http://$nacos_server:8848/nacos/v1/cs/configs?username=nacos&password=nacos&tenant=$tenant&group=Galaxy"
+  local result=$(curl --retry-delay 3 --retry 3 -sw '%{http_code}' -o /dev/null --request POST "$nacos_push_url&dataId=$data_id&appName=$app_name&type=$suffix" --data-urlencode content@"$config_file")
+  if [[ $result -eq '200' ]]; then
+    sleep 2
+    check=$(curl -sw '%{http_code}' -o /dev/null -X GET "$nacos_push_url&dataId=$data_id")
+    #echo "$check $data_id"
+
+    if [[ $check -ne '200' ]]; then
+      shell_result=0
+    fi
+  fi
+}
+
+# 获取本地文件
+function pushConfig() {
+  for config_path in $(readlink -f $config_path_root); do
+    if [ -d $config_path ]; then
+      for file in $(ls $config_path); do
+        config_path_file=$config_path/$file
+        if [ -f $config_path_file ]; then
+          push $config_path_file
+        fi
+      done
+    fi
+    if [ -f $config_path ]; then
+      push $config_path
+    fi
+  done
+}
+
+pushConfig
+echo $shell_result
+
diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/saved-query-scheduler.yml.j2 b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/saved-query-scheduler.yml.j2
new file mode 100644
index 0000000..7f37622
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/templates/saved-query-scheduler.yml.j2
@@ -0,0 +1,126 @@
+#http的端口
+server:
+  port: 9093
+#更新进度条的时间10s
+scan:
+  result:
+    scheduled:
+      plan: 0/15 * * * * ?
+#同时间执行是线程数
+globle:
+  job_thread: 1
+#Hbasehttp的端口
+#Hbase的表名等配置通畅不需要更改
+hbase:
+  table: tsg_galaxy:saved_query_result
+  zookeeper_quorum: "{{ zookeeper_servers }}"
+  zookeeper_property_clientPort: 2181
+  zookeeper_znode_parent: /hbase
+  client_retries_number: 3
+  rpc_timeout: 100000
+  connect_pool: 10
+    #存入Hbase的cell级别生存时间 根据部署环境填写：1.TSG 不设置(永久有效); 2.CN 7(默认7天) 单位：Day
+  cell_ttl_d:
+
+#查询网关ip
+ck:
+  gateway_ip: "{{ vrrp_instance.default.virtual_ipaddress }}:9999"
+
+#zk集群的ip
+zookeeper:
+  connectString: "{{ zookeeper_servers }}"
+#是否启用zookeeper 0启用(集群) 1禁用(单机)
+  open: "{{ zookeeper_open }}"
+  retryCount: 6
+  elapsedTimeMs: 10000
+  sessionTimeoutMs: 50000
+  connectionTimeoutMs: 50000
+  nameSpace: reportservice
+
+#最大连接数
+http:
+  maxTotal: 300
+#并发数
+  defaultMaxPerRoute: 100
+#创建连接的最长时间
+  connectTimeout: 10000
+#从连接池中获取到连接的最长时间
+  connectionRequestTimeout: 10000
+#数据传输的最长时间
+  socketTimeout: 21605000
+#提交请求前测试连接是否可用
+  staleConnectionCheckEnabled: true
+  socketTimeoutShort: 30000
+
+#mariadb的url
+spring:
+  application:
+    name: saved-query-scheduler
+
+  datasource:
+    url: "jdbc:mariadb://{{ vrrp_instance.default.virtual_ipaddress }}:3306/tsg_olap?serverTimezone=UTC&useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false"
+#mariadb的用户名
+    username: "ENC(63aTpwv2vH0vPikW+3Jjig==)"
+#mariadb的密码
+    password: "ENC(LDEb2OekU7iZWiFw6pUYBSozVKP27r1y)"
+
+#以下配置不需要更改通常
+    name: druidDataSource
+    type: com.alibaba.druid.pool.DruidDataSource
+    driver-class-name: org.mariadb.jdbc.Driver
+
+#配置监控统计拦截的filters，去掉后监控界面SQL无法进行统计，’wall’用于防火墙
+    druid:
+      filters: stat,wall,slf4j
+            #最大连接数
+      max-active: 30
+            #最小连接数
+      min-idle: 1
+            #初始化连接数
+      initial-size: 2
+            #获取连接最大超时时间
+      max-wait: 600000
+            #间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒
+      time-between-eviction-runs-millis: 60000
+            # 一个连接在池中最小生存的时间，单位是毫秒
+      min-evictable-idle-time-millis: 300000
+            #验证连接是否可用,在数据库中执行一条sql
+      validation-query: select 1
+            #建议配置为true，不影响性能，并且保证安全性。申请连接的时候检测，如果空闲时间大于timeBetweenEvictionRunsMillis，
+            # 执行validationQuery检测连接是否有效
+      test-while-idle: true
+            #申请连接时执行validationQuery检测连接是否有效，做了这个配置会降低性能
+      test-on-borrow: true
+      test-on-return: false
+      connection-properties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=500
+            #是否开启WebStatFilter
+      web-stat-filter:
+        enabled: true
+            #设置不统计哪些URL(用于排除一些不必要的url)
+        exclusions: '*.js,*.gif,*.jpg,*..ng,*.css,*.ico,/druid/*'
+            #是否开启Druid监控信息显示页面
+      stat-view-servlet:
+        enabled: true
+            #甚至浏览器访问路径
+        url-pattern: /druid/*
+            #禁止手动重置监控数据
+        reset-enable: false
+            #durid-ui页面账户密码
+        login-username: admin
+        login-password: admin
+            #Spring监控，对内部各接口调用的监控,需要导入aop相关包
+      aop-patterns: com.mesa.reportservice.controller.*,com.mesa.reportservice.service.*,com.mesa.reportservice.mapper.*
+mybatis:
+  typeAliasesPackage: com.mesa.reportservice.bean
+  mapperLocations: classpath*:/mappers/*.xml
+management:
+  endpoints:
+    web:
+      exposure:
+        include: '*'
+  metrics:
+    tags:
+      application: saved-query-scheduler
+
+logging:
+  config: ./config/log4j2-dev.xml
diff --git a/tsg-olap/parcels/roles/initialization/saved-query-scheduler/vars/main.yml b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/vars/main.yml
new file mode 100644
index 0000000..a46c2d8
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/saved-query-scheduler/vars/main.yml
@@ -0,0 +1,18 @@
+software_packages_dir: "{{ role_path }}/../software-packages"
+image_name: saved-query-scheduler
+
+image_tag: 374
+
+container_name: "{{ image_name }}"
+
+backup_path: "{{ deploy_dir }}/backup/tsg-olap/{{ old_version }}/{{ container_name }}"
+
+backup_items:
+  - config
+  - docker-compose.yml
+  - init
+
+zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ host }}:2181{% if not loop.last %},{% endif %}{% endfor %}"
+
+#是否启用zookeeper 0启用(集群) 1禁用(单机)
+zookeeper_open: "{{ 0 if groups.saved_query_scheduler | length > 1 else 1 }}"
diff --git a/tsg-olap/parcels/roles/initialization/schedule-service/tasks/init.yml b/tsg-olap/parcels/roles/initialization/schedule-service/tasks/init.yml
new file mode 100644
index 0000000..0bed724
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/schedule-service/tasks/init.yml
@@ -0,0 +1,51 @@
+- name: Create directory
+  file:
+    state: directory
+    path: '{{ deploy_dir }}/schedule-service/{{ item }}'
+  loop:
+    - 'dos-baseline'
+    - 'iplearning'
+    - 'mmdb'
+
+- name: Unpack service to {{ deploy_dir }}/
+  unarchive:
+    src: 'files/{{ item }}'
+    dest: '{{ deploy_dir }}/schedule-service/'
+  loop: 
+    - 'dos-baseline.zip'
+    - 'iplearning.zip'
+
+- name: Copy service files
+  loop:
+    - { src: "iplearning/application.properties.j2", dest: "iplearning/application.properties", mode: "0644" }
+    - { src: "iplearning/start.sh.j2", dest: "iplearning/start.sh", mode: "0755" }
+    - { src: "dos-baseline/application.properties.j2", dest: "dos-baseline/application.properties", mode: "0644" }
+    - { src: "mmdb/mmdb-perl.pl.j2", dest: "mmdb/mmdb-perl.pl", mode: "0755" }
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ deploy_dir }}/schedule-service/{{ item.dest }}"
+    mode: "{{ item.mode }}"
+    force: true
+
+- name: Change mode and execute iplearning upconfig.sh upconfig.sh
+  shell: cd "{{ deploy_dir }}/schedule-service/iplearning/" && chmod +x upconfig.sh && sh upconfig.sh
+
+- name: Change mode and execute dosbaseline upconfig.sh
+  shell: cd "{{ deploy_dir }}/schedule-service/dos-baseline/" && chmod +x upconfig.sh && sh upconfig.sh
+
+- name: Copying perl image
+  copy:
+    src: '{{ software_packages_dir }}/{{ perl_image_name }}-{{ perl_image_tag }}.tar'
+    dest: '{{ deploy_dir }}/schedule-service/mmdb'
+    force: true
+
+- name: Load perl Image
+  docker_image:
+    name: '{{ perl_image_name }}'
+    tag: '{{ perl_image_tag }}'
+    load_path: '{{ deploy_dir }}/schedule-service/mmdb/{{ perl_image_name }}-{{ perl_image_tag }}.tar'
+    source: load
+    force_tag: yes
+    force_source: yes
+    timeout: 300
+
diff --git a/tsg-olap/parcels/roles/initialization/schedule-service/tasks/main.yml b/tsg-olap/parcels/roles/initialization/schedule-service/tasks/main.yml
new file mode 100644
index 0000000..7df2750
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/schedule-service/tasks/main.yml
@@ -0,0 +1,9 @@
+- block:
+    - include: uninstall.yml
+    - include: init.yml
+  when: operation=='install'
+
+- block:
+    - include: init.yml
+  when: operation=='upgrade'
+
diff --git a/tsg-olap/parcels/roles/initialization/schedule-service/tasks/uninstall.yml b/tsg-olap/parcels/roles/initialization/schedule-service/tasks/uninstall.yml
new file mode 100644
index 0000000..d2a9d36
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/schedule-service/tasks/uninstall.yml
@@ -0,0 +1,4 @@
+- name: delete old folders
+  file:
+    state: absent
+    path: '{{ deploy_dir }}/schedule-service'
diff --git a/tsg-olap/parcels/roles/initialization/schedule-service/templates/dos-baseline/application.properties.j2 b/tsg-olap/parcels/roles/initialization/schedule-service/templates/dos-baseline/application.properties.j2
new file mode 100644
index 0000000..5e472e1
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/schedule-service/templates/dos-baseline/application.properties.j2
@@ -0,0 +1,85 @@
+############## 数据库配置 ###############
+##########################################
+#Druid配置
+druid.url=jdbc:avatica:remote:url=http://{{ vrrp_instance.default.virtual_ipaddress }}:8082/druid/v2/sql/avatica/
+druid.driver=org.apache.calcite.avatica.remote.Driver
+druid.table=traffic_top_destination_ip_metrics_log
+
+#HBase配置
+hbase.table=dos:ddos_traffic_baselines
+hbase.zookeeper.quorum={{ zookeeper_servers }}
+hbase.zookeeper.client.port=2181
+
+##########################################
+############## Druid数据读取 ###############
+##########################################
+#读取druid时间范围方式，
+# 0：读取默认范围天数read.historical.days；
+# 1：指定时间范围
+read.druid.time.limit.type=0
+read.druid.min.time=1663430400000
+read.druid.max.time=1663603200000
+
+#Druid字段映射
+druid.attacktype.tcpsynflood=TCP SYN Flood
+druid.attacktype.udpflood=UDP Flood
+druid.attacktype.icmpflood=ICMP Flood
+druid.attacktype.dnsamplification=DNS Flood
+druid.columnname.serverip=destination_ip
+druid.columnname.vsysid=vsys_id
+druid.columnname.attacktype=attack_type
+druid.columnname.recvtime=__time
+druid.columnname.partition.num=partition_num
+baseline.metric.type=session_rate
+#baseline存储级别，1-type1;2-type1及type2;3-all
+baseline.save.level=3
+
+#Hbase字段映射
+hbase.baseline.generation.type.suffix=baseline_type
+hbase.baseline.zero.replace.value.suffix=default_value
+
+#数据情况
+#读取历史N天数据，最小值为3天（需要判断周期性）
+read.historical.days=3
+#历史数据汇聚粒度为10分钟
+historical.grad=10
+# 数据库Time格式
+time.format=yyyy-MM-dd HH:mm:ss
+
+##########################################
+############ Baseline生成参数 #############
+##########################################
+baseline.range.days=1
+baseline.function=KalmanFilter
+baseline.period.correlative.threshold=0.8
+baseline.historical.frequency.thread=0.5
+baseline.exception.percentile=0.99
+baseline.exception.fill.percentile=0.99
+baseline.rational.percentile=0.99
+
+baseline.type2.percentile=1
+baseline.type3.percentile=1
+
+# p99/p50 倍数阈值
+baseline.times.percent.times=10
+
+#Kalman Filter
+baseline.kalman.q=0.1
+baseline.kalman.r=0.1
+baseline.kalman.p=8
+baseline.kalman.m=2
+
+monitor.frequency.bin.num=100
+
+##########################################
+################ 并发参数 #################
+##########################################
+all.partition.num=10
+core.pool.size=10
+max.pool.size=10
+#druid分区字段partition_num的最大值为9999
+druid.statement.query.timeout=36000
+druid.partition.num.max=10000
+druid.connection.retry.time.max=10000
+#druid重连等待时间约为一个线程处理完成时间
+druid.connection.retry.sleep.time=1000
diff --git a/tsg-olap/parcels/roles/initialization/schedule-service/templates/iplearning/application.properties.j2 b/tsg-olap/parcels/roles/initialization/schedule-service/templates/iplearning/application.properties.j2
new file mode 100644
index 0000000..392b6e6
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/schedule-service/templates/iplearning/application.properties.j2
@@ -0,0 +1,45 @@
+#spark任务配置
+spark.sql.shuffle.partitions=30
+spark.executor.memory=4g
+spark.executor.cores=10
+spark.cores.max=10
+spark.local.dir=./tmp
+spark.app.name=iplearning
+spark.network.timeout=300s
+spark.serializer=org.apache.spark.serializer.KryoSerializer
+master=local[*]
+#spark读取clickhouse配置
+spark.read.clickhouse.url=jdbc:clickhouse://{{ vrrp_instance.default.virtual_ipaddress }}:8124/tsg_galaxy_v3
+spark.read.clickhouse.driver=ru.yandex.clickhouse.ClickHouseDriver
+spark.read.clickhouse.user=default
+spark.read.clickhouse.password={{ clickhouse_default_pin }}
+spark.read.clickhouse.numPartitions=5
+spark.read.clickhouse.fetchsize=10000
+spark.read.clickhouse.partitionColumn=LAST_FOUND_TIME
+spark.read.clickhouse.session.table=session_record
+spark.read.clickhouse.radius.table=radius_record
+clickhouse.socket.timeout=300000
+#arangoDB配置
+arangoDB.host={{ groups.arangodb[0] }}
+arangoDB.port=8529
+arangoDB.user=upsert
+arangoDB.password={{ arangodb_upsert_pin }}
+arangoDB.DB.name=tsg_galaxy_v3
+arangoDB.ttl=3600
+
+thread.pool.number=10
+
+#读取clickhouse时间范围方式，0：读取过去一小时；1：指定时间范围
+clickhouse.time.limit.type=0
+read.clickhouse.max.time=1634902508
+read.clickhouse.min.time=1631759985
+
+update.arango.batch=10000
+
+distinct.client.ip.num=10000
+recent.count.hour=24
+
+update.interval=3600
+arangodb.total.num=20000000
+#读取radius时间范围,与radius任务执行周期一致,单位:分钟
+read.radius.granularity=-30
diff --git a/tsg-olap/parcels/roles/initialization/schedule-service/templates/iplearning/start.sh.j2 b/tsg-olap/parcels/roles/initialization/schedule-service/templates/iplearning/start.sh.j2
new file mode 100644
index 0000000..0c30868
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/schedule-service/templates/iplearning/start.sh.j2
@@ -0,0 +1,43 @@
+#!/bin/bash
+source /etc/profile
+
+source /etc/profile
+
+jar_name="ip-learning-spark.jar"
+java_opts="-Xms1g -Xmx5g"
+iplearning_home=/opt/schedule-service/iplearning
+
+isrun=$(pgrep -f ip-learning-spark.jar | wc -l)
+
+#start spark job
+function startJob() {
+        mkdir -p $iplearning_home/logs/IpLearningApplication
+        mkdir -p $iplearning_home/logs/IpRecommendApplication
+
+        echo "IpLearningApplication 程序运行 - $(date "+%Y-%m-%d %H:%M:%S")" >>$iplearning_home/logs/IpLearningApplication/running-$(date +'%Y-%m').log
+        nohup /usr/lib/jvm/jdk1.8.0_73/bin/java -cp $iplearning_home/$jar_name $java_opts cn.ac.iie.main.IpLearningApplication >>$iplearning_home/logs/IpLearningApplication/running-$(date +'%Y-%m').log
+
+        echo "IpRecommendApplication 程序运行 - $(date "+%Y-%m-%d %H:%M:%S")" >>$iplearning_home/logs/IpRecommendApplication/running-$(date +'%Y-%m').log
+        nohup /usr/lib/jvm/jdk1.8.0_73/bin/java -cp $iplearning_home/$jar_name $java_opts cn.ac.iie.main.IpRecommendApplication >>$iplearning_home/logs/IpRecommendApplication/running-$(date +'%Y-%m').log
+
+}
+
+if [[ $isrun -ge 1 ]]; then
+        OLD_NUM=$(cat $iplearning_home/runnum)
+        RESTART_NUM=$(expr $OLD_NUM + 1)
+        echo $RESTART_NUM >$iplearning_home/runnum
+
+        if [ $RESTART_NUM -gt "2" ]; then
+                pgrep -f ip-learning-spark.jar | xargs kill -9
+                startJob
+
+                echo 1 >$iplearning_home/runnum
+        fi
+
+else
+
+        startJob
+        echo 1 >$iplearning_home/runnum
+        nohup $iplearning_home/iplearning_monitor.sh >/dev/null 2>&1 &
+
+fi
diff --git a/tsg-olap/parcels/roles/initialization/schedule-service/templates/mmdb/mmdb-perl.pl.j2 b/tsg-olap/parcels/roles/initialization/schedule-service/templates/mmdb/mmdb-perl.pl.j2
new file mode 100644
index 0000000..2b2180b
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/schedule-service/templates/mmdb/mmdb-perl.pl.j2
@@ -0,0 +1,146 @@
+#!/usr/bin/perl
+use strict;      #use this all times
+use warnings;    #this too - helps a lot!
+use JSON;
+use MaxMind::DB::Writer::Tree;
+use Getopt::Long;
+use utf8;
+use LWP::UserAgent;
+use POSIX qw(strftime);
+binmode(STDIN,":encoding(gbk)");
+
+
+###################连接信息###################
+my $serverHost="{{ cm_api }}";
+my $token="{{ cm_api_token }}";
+
+#自定义：0. 内置：1
+my @args = @ARGV;
+my $isBuiltin= @args?$args[0]:1;
+my $url="http://$serverHost:8080/v1/global/geolocation/dict/detail?is_builtin=$isBuiltin&vsys_vim id=0";
+my $fileName=$isBuiltin==0?'ip_user_defined':'ip_builtin';
+my $outfile="$fileName.mmdb";
+my $mmdbLog="$fileName.log";
+my $mmdbStatus=$fileName.'_status.json';
+print "The perl script is running, output to $outfile \n";
+
+my %types = (
+        AREA_CODE => 'utf8_string',
+        ASN => 'utf8_string',
+        ISP => 'utf8_string',
+        LATITUDE => 'utf8_string',
+        LONGITUDE => 'utf8_string',
+        COUNTRY => 'utf8_string',
+        SUPER_ADMINISTRATIVE_AREA => 'utf8_string',
+        ADMINISTRATIVE_AREA => 'utf8_string',
+        SUB_ADMINISTRATIVE_AREA => 'utf8_string',
+        LOCALITY => 'utf8_string',
+        DEPENDENT_LOCALITY => 'utf8_string',
+        DOUBLE_DEPENDENT_LOCALITY => 'utf8_string',
+);
+
+# V6databse info
+my $treeV6 = MaxMind::DB::Writer::Tree->new(
+       ip_version            => 6,
+       record_size           => 24,
+       database_type         => 'IPLOCATOR',
+       languages             => ['en'],
+       description           => { en => 'My database of IP data' },
+       map_key_type_callback => sub { $types{ $_[0] } },
+       remove_reserved_networks => 0,
+);
+
+my $pageNo = 1;
+my $pageSize = 2000;
+my $total= 0;
+my $requestMsg= '';
+my $requestCode= 0;
+my @ipSample= ();
+
+my $ua = new LWP::UserAgent;
+while(1==1){
+my $request = new HTTP::Request("GET", "$url&page_size=$pageSize&page_no=$pageNo");
+$request->header('Authorization' => $token);
+my $response = $ua->request($request);
+my $json = new JSON;
+my $obj = $json->decode($response->content);
+
+$requestCode=$obj->{"code"};
+$requestMsg=$obj->{"msg"};
+my $dbTotal=$obj->{"data"}->{"total"};
+
+#记录日志
+open(MMDBLOG, "+>>$mmdbLog");
+my $gmtDatestring = strftime "%Y-%m-%d %H:%M:%S", gmtime;
+
+my $list=$obj->{"data"}->{"list"};
+my $JSON =JSON->new->allow_nonref;
+my $data = $JSON->encode($list);
+my $arraydata=decode_json($data);
+foreach my $ref (@$arraydata) {
+        $treeV6->insert_range(
+            $ref->{'start_ip'},
+            $ref->{'end_ip'},
+            {
+            AREA_CODE => isNotBlank($ref->{'areacode'}),
+            ASN       => isNotBlank($ref->{'asnumber'}),
+            ISP       => isNotBlank($ref->{'isp'}),
+            LATITUDE  => isNotBlank($ref->{'latitude'}),
+            LONGITUDE => isNotBlank($ref->{'longitude'}),
+            COUNTRY   => changeSplit(isNotBlank($ref->{'country_region'})),
+            SUPER_ADMINISTRATIVE_AREA => changeSplit(isNotBlank($ref->{'super_administrative_area'})),
+            ADMINISTRATIVE_AREA => changeSplit(isNotBlank($ref->{'administrative_area'})),
+            SUB_ADMINISTRATIVE_AREA => changeSplit(isNotBlank($ref->{'sub_administrative_area'})),
+            LOCALITY => changeSplit(isNotBlank($ref->{'locality'})),
+            DEPENDENT_LOCALITY => changeSplit(isNotBlank($ref->{'dependent_locality'})),
+            DOUBLE_DEPENDENT_LOCALITY => changeSplit(isNotBlank($ref->{'double_dependent_locality'})),
+            },
+        );
+        $total+=1;
+}
+if(@$arraydata){
+  push(@ipSample, @$arraydata[0]->{'start_ip'});
+}
+my $percentage = $dbTotal==0?0 : int(($total/$dbTotal) * 100);
+print "$gmtDatestring MMDB request $requestMsg pageNum: $pageNo, queried/total: $total/$dbTotal, progress: $percentage% .\n";
+print MMDBLOG "$gmtDatestring MMDB request $requestMsg pageNum: $pageNo, queried/total: $total/$dbTotal, progress: $percentage% .\n";
+$pageNo+=1;
+if(@$arraydata < $pageSize){
+    last;
+   }
+}
+
+if($total>0){
+    open my $fhV6, '>:raw', $outfile;
+    $treeV6->write_tree($fhV6);
+    close($outfile);
+}
+
+my %statusHash = ('status'=>$requestCode, 'updateTime'=>time(), 'total'=>$total, 'message'=>$requestMsg,'ipSample'=> \@ipSample);
+my $statusJson = encode_json \%statusHash;
+open my $statusFile, '>', $mmdbStatus;
+print $statusFile $statusJson;
+
+my $gmtDateLog = strftime "%Y-%m-%d %H:%M:%S", gmtimvim m   e;
+print MMDBLOG "$gmtDateLog MMDB IP Write Over. \n\n";
+close(MMDBLOG);
+print "The perl script is finished!\n";
+
+sub changeSplit {
+  my $n = scalar(@_);
+  my $i = $_[0];
+  $i =~ s/\\b/ /g;
+
+  return "$i"
+}
+
+sub isNotBlank {
+   my $tmp = $_[0];
+   if($tmp){
+     return $tmp;
+   } else{
+     return " ";
+   }
+}
+
+
diff --git a/tsg-olap/parcels/roles/initialization/schedule-service/vars/main.yml b/tsg-olap/parcels/roles/initialization/schedule-service/vars/main.yml
new file mode 100644
index 0000000..5cc1bd9
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/schedule-service/vars/main.yml
@@ -0,0 +1,7 @@
+software_packages_dir: "{{ role_path }}/../software-packages"
+
+zk_servers_noport: "{% for host in groups['zookeeper'] %}{{ host }}{% if not loop.last %},{% endif %}{% endfor %}"
+zookeeper_servers: "{% for host in groups['zookeeper'] %}{{ host }}:2181{% if not loop.last %},{% endif %}{% endfor %}"
+
+perl_image_name: perl
+perl_image_tag: v3.1
diff --git a/tsg-olap/parcels/roles/initialization/yarn/tasks/backup.yml b/tsg-olap/parcels/roles/initialization/yarn/tasks/backup.yml
new file mode 100644
index 0000000..1103fa8
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/tasks/backup.yml
@@ -0,0 +1,25 @@
+- name: Check if backup directory exists
+  stat:
+    path: "{{ backup_path }}"
+  register: exist_status
+
+- name: Backup directories
+  block:
+    - name: Creat backup directory
+      file:
+        state: directory
+        path: "{{ backup_path }}"
+
+    - name: Copying directories to backup
+      copy:
+        src: "{{ deploy_dir }}/flink/{{ item }}"
+        dest: "{{ backup_path }}"
+        remote_src: true
+      loop: "{{ backup_items }}"
+
+  rescue:
+    - name: Remove backup directory on failure
+      file:
+        path: "{{ backup_path }}"
+        state: absent
+  when: exist_status.stat.exists == false
diff --git a/tsg-olap/parcels/roles/initialization/yarn/tasks/bak.check.yml b/tsg-olap/parcels/roles/initialization/yarn/tasks/bak.check.yml
new file mode 100644
index 0000000..65ff569
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/tasks/bak.check.yml
@@ -0,0 +1,27 @@
+- block:
+    - name: Checking the number of Flink tasks
+      shell: "source /etc/profile && sleep 30 && yarn application --list | grep -v 'Total number of applications' | grep RUNNING | wc -l"
+      register: jobNums
+    
+    - name: Checking the number of Flink tasks
+      fail:
+        msg: "The number of Flink running tasks does not match the record.The browser enter {{ groups.yarn[0] }}:8080 for verification."
+      when: jobNums.stdout != '11'
+  run_once: true
+  delegate_facts: true
+  delegate_to: "{{ groups.yarn[0] }}"
+  when: (groups.yarn|length ) > 1
+  
+- block:
+    - name: Checking the number of Flink tasks
+      shell: "source /etc/profile && sleep 30 && flink list | grep '(RUNNING)' | wc -l"
+      register: jobNums
+    
+    - name: Checking the number of Flink tasks
+      fail:
+        msg: "The number of Flink running tasks does not match the record.The browser enter {{ groups.yarn[0] }}:8080 for verification."
+      when: jobNums.stdout != '11'
+  run_once: true
+  delegate_facts: true
+  delegate_to: "{{ groups.yarn[0] }}"
+  when: (groups.yarn|length ) == 1
diff --git a/tsg-olap/parcels/roles/initialization/yarn/tasks/check.yml b/tsg-olap/parcels/roles/initialization/yarn/tasks/check.yml
new file mode 100644
index 0000000..020e1dd
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/tasks/check.yml
@@ -0,0 +1,65 @@
+- name: Get init task template
+  find:
+    paths: "templates"
+    patterns: "config"
+    file_type: directory
+    recurse: yes
+  register: find_dirctory
+  run_once: true
+  delegate_facts: true
+  delegate_to: 127.0.0.1
+
+- set_fact:
+    find_config: "{{ find_dirctory.files | map(attribute='path') | list }}"
+
+- name: Find config template
+  find:
+    paths: "{{ item }}"
+    file_type: "file"
+    recurse: yes
+  with_list: "{{ find_config }}"
+  register: find_file
+  run_once: true
+  delegate_facts: true
+  delegate_to: 127.0.0.1
+
+- set_fact:
+    init_task_list: "{{ find_file.results | json_query('[].files[].path') | map('basename') | map('regex_replace', '^(.*)\\.j2$', '\\1') | list }}"
+
+- block:
+  - name: Gets the running yarn task
+    shell: "source /etc/profile && sleep 60 && yarn application --list -appStates RUNNING | grep -v 'Total number of applications' | grep RUNNING| awk '{print $2}'"
+    register: run_task_list
+
+  - debug: var=run_task_list.stdout_lines
+
+  - name: Checking the number of Flink task
+    fail:
+      msg: "The number of Flink running tasks does not match the record.The browser enter {{ groups.yarn[0] }}:8080 for verification."
+    when: run_task_list.stdout_lines | intersect(init_task_list) |length != (init_task_list|length)
+  run_once: true
+  delegate_facts: true
+  delegate_to: "{{ groups.yarn[0] }}"
+  when: task_mode == 'yarn-per-job'
+
+- block:
+  - name: Gets the running yarn task
+    shell: source /etc/profile && sleep 60 && yarn application --list -appStates RUNNING | grep -v 'Total number of applications' | grep RUNNING| awk '{print $1}'
+    register: run_application_id
+
+  - debug: var=run_application_id.stdout
+
+  - name: Gets the running yarn task
+    shell: curl -s http://{{ groups.yarn[0] }}:8080/proxy/{{ run_application_id.stdout }}/jobs/overview| jq -r '.jobs[] | select(.state=="RUNNING") | .name'
+    register: run_task_list
+
+  - debug: var=run_task_list.stdout_lines
+
+  - name: Checking the number of Flink task
+    fail:
+      msg: "The number of Flink running tasks does not match the record.The browser enter {{ groups.yarn[0] }}:8080 for verification."
+    when: run_task_list.stdout_lines | intersect(init_task_list) |length != (init_task_list|length)
+  run_once: true
+  delegate_facts: true
+  delegate_to: "{{ groups.yarn[0] }}"
+  when: task_mode == 'yarn-session'
diff --git a/tsg-olap/parcels/roles/initialization/yarn/tasks/init.yml b/tsg-olap/parcels/roles/initialization/yarn/tasks/init.yml
new file mode 100644
index 0000000..d400056
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/tasks/init.yml
@@ -0,0 +1,154 @@
+- name: Set tasks directory variable
+  set_fact:
+    task_dir:
+      - app-protocol-stat-traffic-merge
+      - dos-detection
+      - file-chunk-combiner
+      - groot-stream
+      - sip-rtp-correlation
+      - topn-metrics-job
+    sub_dir:
+      - bin
+      - config
+      - env
+      - lib
+
+- name: Create yarn session
+  shell: source /etc/profile && yarn-session.sh -d
+  run_once: true
+  when: task_mode == 'yarn-session'
+ 
+- name: Create {{ deploy_dir }}/flink/topology
+  file:
+    path: "{{ deploy_dir }}/flink/topology"
+    state: directory
+
+- name: Unpack Knowledge base files
+  unarchive:
+    src: 'files/data.zip'
+    dest: '{{ deploy_dir }}/flink/topology/'
+
+- block: 
+    - name: Create {{ deploy_dir }}/flink/topology
+      file:
+        path: "{{ deploy_dir }}/flink/topology/{{ item.0 }}/{{ item.1 }}"
+        state: directory
+      with_nested:
+        - "{{ task_dir }}"
+        - "{{ sub_dir }}"
+      changed_when: false
+
+    - name: Unpack Knowledge base files
+      unarchive:
+        src: 'files/data.zip'
+        dest: '{{ deploy_dir }}/flink/topology/'
+
+    - name: Copying upload_knowlodge.sh
+      template:
+        src: 'upload_knowlodge.sh.j2'
+        dest: '{{ deploy_dir }}/flink/topology/upload_knowlodge.sh'
+        force: true
+        mode: 0755
+
+    - name: upload Knowledge
+      shell: cd {{ deploy_dir }}/flink/topology/ && sh upload_knowlodge.sh
+      run_once: true
+
+    - name: Copying topology to {{ deploy_dir }}/
+      copy:
+        src: 'files/topology'
+        dest: '{{ deploy_dir }}/flink/'
+        force: true
+  
+    - name: Copying jobs config files
+      template:
+        src: "{{ item.name }}/config/{{ item.config }}.j2"
+        dest: "{{ deploy_dir }}/flink/topology/{{ item.name }}/config/{{ item.config }}"
+        force: true
+      with_items: 
+        - { name: 'app-protocol-stat-traffic-merge', config: 'agg_app_protocol_traffic' }
+        - { name: 'dos-detection', config: 'detection_dos_attack' }
+        - { name: 'file-chunk-combiner', config: 'agg_traffic_file_chunk_combine' }
+        - { name: 'file-chunk-combiner', config: 'agg_troubleshooting_file_chunk_combine' }
+        - { name: 'sip-rtp-correlation', config: 'correlation_sip_rtp_session' }
+        - { name: 'topn-metrics-job', config: 'agg_session_record_topn' }
+
+    - name: Copying jobs env files
+      template:
+        src: "{{ item.name }}/env/{{ item.config }}.sh"
+        dest: "{{ deploy_dir }}/flink/topology/{{ item.name }}/env/{{ item.config }}.sh"
+        mode: 0755
+        force: true
+      with_items: 
+        - { name: 'app-protocol-stat-traffic-merge', config: 'agg_app_protocol_traffic' }
+        - { name: 'dos-detection', config: 'detection_dos_attack' }
+        - { name: 'file-chunk-combiner', config: 'agg_traffic_file_chunk_combine' }
+        - { name: 'file-chunk-combiner', config: 'agg_troubleshooting_file_chunk_combine' }
+        - { name: 'sip-rtp-correlation', config: 'correlation_sip_rtp_session' }
+        - { name: 'topn-metrics-job', config: 'agg_session_record_topn' }
+
+    - name: Copying jobs start.sh
+      template:
+        src: "start.sh"
+        dest: "{{ deploy_dir }}/flink/topology/{{ item.name }}/bin/start.sh"
+        force: true
+        mode: 0755
+      with_items:
+        - { name: 'app-protocol-stat-traffic-merge' }
+        - { name: 'dos-detection' }
+        - { name: 'file-chunk-combiner' }
+        - { name: 'sip-rtp-correlation' }
+        - { name: 'topn-metrics-job' }
+
+    - name: Copying jobs stop.sh
+      template:
+        src: "stop.sh"
+        dest: "{{ deploy_dir }}/flink/topology/{{ item.name }}/bin/stop.sh"
+        force: true
+        mode: 0755
+      with_items:
+        - { name: 'app-protocol-stat-traffic-merge' }
+        - { name: 'dos-detection' }
+        - { name: 'file-chunk-combiner' }
+        - { name: 'groot-stream' }
+        - { name: 'sip-rtp-correlation' }
+        - { name: 'topn-metrics-job' }
+
+    - name: Copying groot start.sh
+      template:
+        src: "{{ item.name }}/start.sh"
+        dest: "{{ deploy_dir }}/flink/topology/{{ item.name }}/bin/start.sh"
+        force: true
+        mode: 0755
+      with_items:
+        - { name: 'groot-stream' }
+
+    - name: Copying groot config templates
+      template:
+        src: "{{ item }}"
+        dest: "{{ deploy_dir }}/flink/topology/groot-stream/config/{{ item | basename | regex_replace('.j2$', '') }}"
+        mode: "0644"
+        force: true
+      with_fileglob: "templates/groot-stream/config/*"
+      
+    - name: Copying groot env templates
+      template:
+        src: "{{ item }}"
+        dest: "{{ deploy_dir }}/flink/topology/groot-stream/env/{{ item | basename }}"
+        mode: "0755"
+        force: true
+      with_fileglob: "templates/groot-stream/env/*"
+
+    - name: Starting flink tasks
+      shell: cd '{{ deploy_dir }}/flink/topology/{{ item.name }}/bin' && chmod +x *.sh && ./stop.sh && ./start.sh
+      with_items:
+        - { name: 'groot-stream' }
+        - { name: 'app-protocol-stat-traffic-merge' }
+        - { name: 'dos-detection' }
+        - { name: 'file-chunk-combiner' }
+        - { name: 'sip-rtp-correlation' }
+        - { name: 'topn-metrics-job' }
+      run_once: true
+
+  when: inventory_hostname in groups['yarn'][:2]
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/tasks/main.yml b/tsg-olap/parcels/roles/initialization/yarn/tasks/main.yml
new file mode 100644
index 0000000..769fd2b
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/tasks/main.yml
@@ -0,0 +1,16 @@
+- block:
+  - include: uninstall.yml
+  - include: init.yml
+  - include: check.yml
+  when: (operation) == "install"
+
+- block:
+  - include: backup.yml
+    when: inventory_hostname in groups['yarn'][:2]
+  - include: uninstall.yml
+  - include: init.yml
+  when: (operation) == "upgrade"
+
+- block:
+  - include: uninstall.yml
+  when: (operation) == "uninstall" and inventory_hostname in groups['yarn'][:2]
diff --git a/tsg-olap/parcels/roles/initialization/yarn/tasks/uninstall.yml b/tsg-olap/parcels/roles/initialization/yarn/tasks/uninstall.yml
new file mode 100644
index 0000000..4860ee0
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/tasks/uninstall.yml
@@ -0,0 +1,14 @@
+- name: Check {{ deploy_dir }}/flink/topology
+  stat:
+    path: "{{ deploy_dir }}/flink/topology"
+  register: directory_stat
+
+- name: Stop all jobs
+  raw: yarn application -list | awk '/application_/ {print $1}' | xargs -I {} yarn application -kill {}
+  run_once: true
+
+- name: Delete {{ deploy_dir }}/flink/topology
+  file:
+     path: "{{ deploy_dir }}/flink/topology"
+     state: absent
+  when: directory_stat.stat.exists
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/app-protocol-stat-traffic-merge/config/agg_app_protocol_traffic.j2 b/tsg-olap/parcels/roles/initialization/yarn/templates/app-protocol-stat-traffic-merge/config/agg_app_protocol_traffic.j2
new file mode 100644
index 0000000..a102c7b
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/app-protocol-stat-traffic-merge/config/agg_app_protocol_traffic.j2
@@ -0,0 +1,27 @@
+#kafka 接收数据topic
+source.kafka.topic=NETWORK-TRAFFIC-METRIC
+
+source.kafka.props.bootstrap.servers={{ kafka_source_servers }}
+
+source.kafka.props.group.id=app-protocol-merge-230510-1
+
+source.kafka.props.security.protocol=SASL_PLAINTEXT
+
+source.kafka.props.sasl.mechanism=PLAIN
+
+source.kafka.props.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019";
+
+#补全数据 输出 topic
+sink.kafka.topic=NETWORK-TRAFFIC-METRIC
+
+sink.kafka.props.bootstrap.servers={{ kafka_sink_servers }}
+
+sink.kafka.props.security.protocol=SASL_PLAINTEXT
+
+sink.kafka.props.sasl.mechanism=PLAIN
+
+sink.kafka.props.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019";
+
+count.window.time=5
+
+watermark.max.orderness=5
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/app-protocol-stat-traffic-merge/env/agg_app_protocol_traffic.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/app-protocol-stat-traffic-merge/env/agg_app_protocol_traffic.sh
new file mode 100644
index 0000000..b3e1b15
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/app-protocol-stat-traffic-merge/env/agg_app_protocol_traffic.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+export MAIN_CLASS="com.zdjizhi.topology.ApplicationProtocolTopology"
+export PARALLELISM="{{ flink.agg_app_protocol_traffic['parallelism'] }}"
+export TASK_MODE="{{ task_mode }}"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+{% for item in flink.agg_app_protocol_traffic | dict2items %}
+{% if item.key != 'parallelism' %}
+-D{{ item.key }}={{ item.value }}
+{% endif %}
+{% endfor %}
+"
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/dos-detection/config/detection_dos_attack.j2 b/tsg-olap/parcels/roles/initialization/yarn/templates/dos-detection/config/detection_dos_attack.j2
new file mode 100644
index 0000000..85bc166
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/dos-detection/config/detection_dos_attack.j2
@@ -0,0 +1,50 @@
+#kafka source
+source.kafka.topic=DOS-SKETCH-RECORD
+source.kafka.props.bootstrap.servers={{ kafka_source_servers }}
+source.kafka.props.group.id=dos-detection-job-20240116
+source.kafka.props.session.timeout.ms=60000
+source.kafka.props.max.poll.records=5000
+source.kafka.props.max.partition.fetch.bytes=31457280
+source.kafka.props.security.protocol=SASL_PLAINTEXT
+source.kafka.props.sasl.mechanism=PLAIN
+source.kafka.props.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin"password="galaxy2019";
+
+#kafka sink
+kafka.sink.event.topic.name=DOS-EVENT
+kafka.sink.metric.topic=TRAFFIC-TOP-DESTINATION-IP-METRIC
+sink.kafka.props.bootstrap.servers={{ kafka_sink_servers }}
+sink.kafka.props.security.protocol=SASL_PLAINTEXT
+sink.kafka.props.sasl.mechanism=PLAIN
+sink.kafka.props.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019";
+sink.kafka.props.acks=1
+sink.kafka.props.retries=0
+sink.kafka.props.linger.ms=10
+sink.kafka.props.request.timeout.ms=30000
+sink.kafka.props.batch.size=262144
+sink.kafka.props.buffer.memory=134217728
+sink.kafka.props.max.request.size=10485760
+sink.kafka.props.compression.type=snappy
+
+#zookeeper地址
+hbase.zookeeper.quorum={{ zookeeper_servers }}
+
+flink.watermark.max.orderness=300
+
+#计算窗口大小，默认600s
+flink.window.max.time=600
+
+#cm服务访问地址
+bifang.server.uri=http://{{ cm_api }}
+
+knowledge.base.uri=http://{{ vrrp_instance.default.virtual_ipaddress }}:9999
+############################## 阈值 配置 ######################################
+static.sensitivity.threshold=1
+#基线敏感阈值
+baseline.sensitivity.threshold=0.2
+#基于baseline判定dos攻击的上下限
+baseline.sessions.minor.threshold=0.2
+baseline.sessions.warning.threshold=1
+baseline.sessions.major.threshold=2.5
+baseline.sessions.severe.threshold=5
+baseline.sessions.critical.threshold=8
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/dos-detection/env/detection_dos_attack.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/dos-detection/env/detection_dos_attack.sh
new file mode 100644
index 0000000..1e478c9
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/dos-detection/env/detection_dos_attack.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+export MAIN_CLASS="com.zdjizhi.main.DosDetectionApplication"
+export PARALLELISM="{{ flink.detection_dos_attack['parallelism'] }}"
+export TASK_MODE="{{ task_mode }}"
+
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+{% for item in flink.detection_dos_attack | dict2items %}
+{% if item.key != 'parallelism' %}
+-D{{ item.key }}={{ item.value }}
+{% endif %}
+{% endfor %}
+"
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/config/agg_traffic_file_chunk_combine.j2 b/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/config/agg_traffic_file_chunk_combine.j2
new file mode 100644
index 0000000..c2fb093
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/config/agg_traffic_file_chunk_combine.j2
@@ -0,0 +1,41 @@
+flink.job.name=agg_traffic_file_chunk_combine
+
+#source相关配置
+source.kafka.parallelism=1
+#9092为无验证 9095为ssl 9094为sasl
+source.kafka.broker={{ kafka_source_servers }}
+source.kafka.group.id=agg_traffic_file_chunk_combine
+source.kafka.topic=TRAFFIC-FILE-STREAM-RECORD
+#earliest从头开始 latest最新
+source.kafka.auto.offset.reset=latest
+source.kafka.session.timeout.ms=60000
+#每次拉取操作从分区中获取的最大记录数
+source.kafka.max.poll.records=1000
+#消费者从单个分区中一次性获取的最大字节数
+source.kafka.max.partition.fetch.bytes=31457280
+source.kafka.enable.auto.commit=true
+#kafka SASL验证用户名
+source.kafka.user=admin
+#kafka SASL及SSL验证密码
+source.kafka.pin=galaxy2019
+#SSL需要
+source.kafka.tools.library={{ deploy_dir }}/topology/data/
+
+parse.message.pack.parallelism=1
+#窗口相关配置
+combiner.window.parallelism=1
+combiner.window.time=10
+combiner.window.idle.time=5
+combiner.window.key.max.chunk=100000
+
+#hos sink相关配置
+sink.hos.parallelism=1
+sink.hos.endpoint=http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos
+sink.hos.bucket=traffic_file_bucket
+sink.hos.token={{ hos_token }}
+sink.hos.http.error.retry=3
+sink.hos.http.max.total=2000
+sink.hos.http.max.per.route=1000
+sink.hos.http.connect.timeout=10000
+sink.hos.http.request.timeout=10000
+sink.hos.http.socket.timeout=60000
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/config/agg_troubleshooting_file_chunk_combine.j2 b/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/config/agg_troubleshooting_file_chunk_combine.j2
new file mode 100644
index 0000000..af3c767
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/config/agg_troubleshooting_file_chunk_combine.j2
@@ -0,0 +1,41 @@
+flink.job.name=agg_troubleshooting_file_chunk_combine
+
+#source相关配置
+source.kafka.parallelism=1
+#9092为无验证 9095为ssl 9094为sasl
+source.kafka.broker={{ kafka_source_servers }}
+source.kafka.group.id=agg_troubleshooting_file_chunk_combine
+source.kafka.topic=TROUBLESHOOTING-FILE-STREAM-RECORD
+#earliest从头开始 latest最新
+source.kafka.auto.offset.reset=latest
+source.kafka.session.timeout.ms=60000
+#每次拉取操作从分区中获取的最大记录数
+source.kafka.max.poll.records=100
+#消费者从单个分区中一次性获取的最大字节数
+source.kafka.max.partition.fetch.bytes=31457280
+source.kafka.enable.auto.commit=true
+#kafka SASL验证用户名
+source.kafka.user=admin
+#kafka SASL及SSL验证密码
+source.kafka.pin=galaxy2019
+#SSL需要
+source.kafka.tools.library={{ deploy_dir }}/topology/data/
+
+parse.message.pack.parallelism=1
+#窗口相关配置
+combiner.window.parallelism=1
+combiner.window.time=5
+combiner.window.idle.time=5
+combiner.window.key.max.chunk=100000
+
+#hos sink相关配置
+sink.hos.parallelism=1
+sink.hos.endpoint=http://{{ vrrp_instance.oss.virtual_ipaddress }}:9098/hos
+sink.hos.bucket=troubleshooting_file_bucket
+sink.hos.token={{ hos_token }}
+sink.hos.http.error.retry=3
+sink.hos.http.max.total=2000
+sink.hos.http.max.per.route=1000
+sink.hos.http.connect.timeout=10000
+sink.hos.http.request.timeout=10000
+sink.hos.http.socket.timeout=60000
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/env/agg_traffic_file_chunk_combine.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/env/agg_traffic_file_chunk_combine.sh
new file mode 100644
index 0000000..551535f
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/env/agg_traffic_file_chunk_combine.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+export MAIN_CLASS="com.zdjizhi.FileChunkCombiner"
+export PARALLELISM="{{ flink.agg_traffic_file_chunk_combine['parallelism'] }}"
+export TASK_MODE="{{ task_mode }}"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+{% for item in flink.agg_traffic_file_chunk_combine | dict2items %}
+{% if item.key != 'parallelism' %}
+-D{{ item.key }}={{ item.value }}
+{% endif %}
+{% endfor %}
+"
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/env/agg_troubleshooting_file_chunk_combine.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/env/agg_troubleshooting_file_chunk_combine.sh
new file mode 100644
index 0000000..e01f666
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/file-chunk-combiner/env/agg_troubleshooting_file_chunk_combine.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+export MAIN_CLASS="com.zdjizhi.FileChunkCombiner"
+
+export PARALLELISM="{{ flink.agg_troubleshooting_file_chunk_combine['parallelism'] }}"
+export TASK_MODE="{{ task_mode }}"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+{% for item in flink.agg_troubleshooting_file_chunk_combine | dict2items %}
+{% if item.key != 'parallelism' %}
+-D{{ item.key }}={{ item.value }}
+{% endif %}
+{% endfor %}
+"
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/dos_event_kafka_to_clickhouse b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/dos_event_kafka_to_clickhouse
new file mode 100644
index 0000000..1ab988c
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/dos_event_kafka_to_clickhouse
@@ -0,0 +1,53 @@
+sources:
+  kafka_source:
+    type : kafka
+    # fields: # [array of object] Field List, if not set, all fields(Map<String, Object>) will be output.
+    # watermark_timestamp: common_recv_time # [string] Watermark Field Name
+    # watermark_timestamp_unit: ms # [string] Watermark Unit, default is ms
+    # watermark_lag: 60 # [number] Watermark Lag, default is 60
+    properties:
+      topic: DOS-EVENT
+      kafka.bootstrap.servers: {{ kafka_sink_servers }}
+      kafka.session.timeout.ms: 60000
+      kafka.max.poll.records: 3000
+      kafka.max.partition.fetch.bytes: 31457280
+      kafka.security.protocol: SASL_PLAINTEXT
+      kafka.ssl.keystore.location:
+      kafka.ssl.keystore.password:
+      kafka.ssl.truststore.location:
+      kafka.ssl.truststore.password:
+      kafka.ssl.key.password:
+      kafka.sasl.mechanism: PLAIN
+      kafka.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019";
+      kafka.buffer.memory:
+      kafka.group.id: dos_event_kafka_to_clickhouse-20231221
+      kafka.auto.offset.reset: latest
+      kafka.max.request.size:
+      kafka.compression.type: none
+      format: json
+
+
+sinks:
+  clickhouse_sink:
+    type: clickhouse
+    properties:
+      host: {{ clickhouse_servers }}
+      table: tsg_galaxy_v3.dos_event_local
+      batch.size: 100000
+      batch.interval: 30s
+      connection.user: default
+      connection.password: galaxy2019
+
+
+application:
+
+  env: # [object] Environment Variables
+    name: dos_event_kafka_to_clickhouse # [string] Job Name
+    pipeline:
+      object-reuse: true # [boolean] Object Reuse, default is false
+  topology:
+    - name: kafka_source
+      downstream: [clickhouse_sink]
+    - name: clickhouse_sink
+
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_proxy_event_kafka_to_clickhouse b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_proxy_event_kafka_to_clickhouse
new file mode 100644
index 0000000..a09b036
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_proxy_event_kafka_to_clickhouse
@@ -0,0 +1,161 @@
+sources:
+  kafka_source:
+    type : kafka
+    # fields: # [array of object] Field List, if not set, all fields(Map<String, Object>) will be output.
+    # watermark_timestamp: common_recv_time # [string] Watermark Field Name
+    # watermark_timestamp_unit: ms # [string] Watermark Unit, default is ms
+    # watermark_lag: 60 # [number] Watermark Lag, default is 60
+    properties:
+      topic: PROXY-EVENT
+      kafka.bootstrap.servers: {{ kafka_sink_servers }}
+      kafka.session.timeout.ms: 60000
+      kafka.max.poll.records: 3000
+      kafka.max.partition.fetch.bytes: 31457280
+      kafka.security.protocol: SASL_PLAINTEXT
+      kafka.ssl.keystore.location:
+      kafka.ssl.keystore.password:
+      kafka.ssl.truststore.location:
+      kafka.ssl.truststore.password:
+      kafka.ssl.key.password:
+      kafka.sasl.mechanism: PLAIN
+      kafka.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019";
+      kafka.buffer.memory:
+      kafka.group.id: etl_proxy_event_kafka_to_clickhouse-20231221
+      kafka.auto.offset.reset: latest
+      kafka.max.request.size:
+      kafka.compression.type: none
+      format: json
+
+processing_pipelines:
+  etl_processor: # [object] Processing Pipeline
+    type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl
+    remove_fields:
+    output_fields:
+    properties:
+      key: value
+    functions: # [array of object] Function List
+
+      - function: ASN_LOOKUP
+        lookup_fields: [ server_ip ]
+        output_fields: [ server_asn ]
+        parameters:
+          option: IP_TO_ASN
+          kb_name: tsg_ip_asn
+
+      - function: ASN_LOOKUP
+        lookup_fields: [ client_ip ]
+        output_fields: [ client_asn ]
+        parameters:
+          option: IP_TO_ASN
+          kb_name: tsg_ip_asn
+
+      - function: SNOWFLAKE_ID
+        lookup_fields: [ '' ]
+        output_fields: [ log_id ]
+        parameters:
+          data_center_id_num: 1
+
+      - function: JSON_EXTRACT
+        lookup_fields: [ device_tag ]
+        output_fields: [ data_center ]
+        filter:
+        parameters:
+          value_expression: $.tags[?(@.tag=='data_center')][0].value
+
+      - function: JSON_EXTRACT
+        lookup_fields: [ device_tag ]
+        output_fields: [ device_group ]
+        filter:
+        parameters:
+          value_expression: $.tags[?(@.tag=='device_group')][0].value
+
+      - function: CURRENT_UNIX_TIMESTAMP
+        output_fields: [ processing_time ]
+        parameters:
+          precision: seconds
+
+      - function: UNIX_TIMESTAMP_CONVERTER
+        lookup_fields: [ __timestamp ]
+        output_fields: [ recv_time ]
+        parameters:
+          precision: seconds
+
+      - function: EVAL
+        output_fields: [ ingestion_time ]
+        parameters:
+          value_expression: 'recv_time'
+
+      - function: DOMAIN
+        lookup_fields: [ http_host,ssl_sni,dtls_sni,quic_sni ]
+        output_fields: [ server_domain ]
+        parameters:
+          option: FIRST_SIGNIFICANT_SUBDOMAIN
+
+      - function: BASE64_DECODE_TO_STRING
+        output_fields: [mail_subject]
+        parameters:
+          value_field: mail_subject
+          charset_field: mail_subject_charset
+
+      - function: BASE64_DECODE_TO_STRING
+        output_fields: [mail_attachment_name]
+        parameters:
+          value_field: mail_attachment_name
+          charset_field: mail_attachment_name_charset
+
+      - function: PATH_COMBINE
+        lookup_fields: [ rtp_pcap_path ]
+        output_fields: [ rtp_pcap_path ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, rtp_pcap_path ]
+
+      - function: PATH_COMBINE
+        lookup_fields: [ http_request_body ]
+        output_fields: [ http_request_body ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, http_request_body ]
+
+      - function: PATH_COMBINE
+        lookup_fields: [ http_response_body ]
+        output_fields: [ http_response_body ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, http_response_body ]
+
+      - function: PATH_COMBINE
+        lookup_fields: [ mail_eml_file ]
+        output_fields: [ mail_eml_file ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, mail_eml_file ]
+
+      - function: PATH_COMBINE
+        lookup_fields: [ packet_capture_file ]
+        output_fields: [ packet_capture_file ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, packet_capture_file ]
+
+sinks:
+  clickhouse_sink:
+    type: clickhouse
+    properties:
+      host: {{ clickhouse_servers }}
+      table: tsg_galaxy_v3.proxy_event_local
+      batch.size: 100000
+      batch.interval: 30s
+      connection.user: default
+      connection.password: galaxy2019
+
+
+application:
+
+  env: # [object] Environment Variables
+    name: etl_proxy_event_kafka_to_clickhouse # [string] Job Name
+    pipeline:
+      object-reuse: true # [boolean] Object Reuse, default is false
+  topology:
+    - name: kafka_source
+      downstream: [etl_processor]
+    - name: etl_processor
+      downstream: [clickhouse_sink]
+    - name: clickhouse_sink
+
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_session_record_kafka_to_clickhouse b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_session_record_kafka_to_clickhouse
new file mode 100644
index 0000000..9812e0b
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_session_record_kafka_to_clickhouse
@@ -0,0 +1,161 @@
+sources:
+  kafka_source:
+    type : kafka
+    # fields: # [array of object] Field List, if not set, all fields(Map<String, Object>) will be output.
+    # watermark_timestamp: common_recv_time # [string] Watermark Field Name
+    # watermark_timestamp_unit: ms # [string] Watermark Unit, default is ms
+    # watermark_lag: 60 # [number] Watermark Lag, default is 60
+    properties:
+      topic:  SESSION-RECORD
+      kafka.bootstrap.servers: {{ kafka_sink_servers }}
+      kafka.session.timeout.ms: 60000
+      kafka.max.poll.records: 3000
+      kafka.max.partition.fetch.bytes: 31457280
+      kafka.security.protocol: SASL_PLAINTEXT
+      kafka.ssl.keystore.location:
+      kafka.ssl.keystore.password:
+      kafka.ssl.truststore.location:
+      kafka.ssl.truststore.password:
+      kafka.ssl.key.password:
+      kafka.sasl.mechanism: PLAIN
+      kafka.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019";
+      kafka.buffer.memory:
+      kafka.group.id: etl_session_record_kafka_to_clickhouse-20231229
+      kafka.auto.offset.reset: latest
+      kafka.max.request.size:
+      kafka.compression.type: none
+      format: json
+
+processing_pipelines:
+  etl_processor: # [object] Processing Pipeline
+    type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl
+    remove_fields:
+    output_fields:
+    properties:
+      key: value
+    functions: # [array of object] Function List
+
+      - function: ASN_LOOKUP
+        lookup_fields: [ server_ip ]
+        output_fields: [ server_asn ]
+        parameters:
+          option: IP_TO_ASN
+          kb_name: tsg_ip_asn
+
+      - function: ASN_LOOKUP
+        lookup_fields: [ client_ip ]
+        output_fields: [ client_asn ]
+        parameters:
+          option: IP_TO_ASN
+          kb_name: tsg_ip_asn
+
+      - function: SNOWFLAKE_ID
+        lookup_fields: [ '' ]
+        output_fields: [ log_id ]
+        parameters:
+          data_center_id_num: 1
+
+      - function: JSON_EXTRACT
+        lookup_fields: [ device_tag ]
+        output_fields: [ data_center ]
+        filter:
+        parameters:
+          value_expression: $.tags[?(@.tag=='data_center')][0].value
+
+      - function: JSON_EXTRACT
+        lookup_fields: [ device_tag ]
+        output_fields: [ device_group ]
+        filter:
+        parameters:
+          value_expression: $.tags[?(@.tag=='device_group')][0].value
+
+      - function: CURRENT_UNIX_TIMESTAMP
+        output_fields: [ processing_time ]
+        parameters:
+          precision: seconds
+
+      - function: UNIX_TIMESTAMP_CONVERTER
+        lookup_fields: [ __timestamp ]
+        output_fields: [ recv_time ]
+        parameters:
+          precision: seconds
+
+      - function: EVAL
+        output_fields: [ ingestion_time ]
+        parameters:
+          value_expression: 'recv_time'
+
+      - function: DOMAIN
+        lookup_fields: [ http_host,ssl_sni,dtls_sni,quic_sni ]
+        output_fields: [ server_domain ]
+        parameters:
+          option: FIRST_SIGNIFICANT_SUBDOMAIN
+
+      - function: BASE64_DECODE_TO_STRING
+        output_fields: [mail_subject]
+        parameters:
+          value_field: mail_subject
+          charset_field: mail_subject_charset
+
+      - function: BASE64_DECODE_TO_STRING
+        output_fields: [mail_attachment_name]
+        parameters:
+          value_field: mail_attachment_name
+          charset_field: mail_attachment_name_charset
+
+      - function: PATH_COMBINE
+        lookup_fields: [ rtp_pcap_path ]
+        output_fields: [ rtp_pcap_path ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, rtp_pcap_path ]
+
+      - function: PATH_COMBINE
+        lookup_fields: [ http_request_body ]
+        output_fields: [ http_request_body ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, http_request_body ]
+
+      - function: PATH_COMBINE
+        lookup_fields: [ http_response_body ]
+        output_fields: [ http_response_body ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, http_response_body ]
+
+      - function: PATH_COMBINE
+        lookup_fields: [ mail_eml_file ]
+        output_fields: [ mail_eml_file ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, mail_eml_file ]
+
+      - function: PATH_COMBINE
+        lookup_fields: [ packet_capture_file ]
+        output_fields: [ packet_capture_file ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, packet_capture_file ]
+
+sinks:
+  clickhouse_sink:
+    type: clickhouse
+    properties:
+      host: {{ clickhouse_servers }}
+      table: tsg_galaxy_v3.session_record_local
+      batch.size: 100000
+      batch.interval: 30s
+      connection.user: default
+      connection.password: galaxy2019
+
+
+application:
+
+  env: # [object] Environment Variables
+    name: etl_session_record_kafka_to_clickhouse # [string] Job Name
+    pipeline:
+      object-reuse: true # [boolean] Object Reuse, default is false
+  topology:
+    - name: kafka_source
+      downstream: [etl_processor]
+    - name: etl_processor
+      downstream: [clickhouse_sink]
+    - name: clickhouse_sink
+
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_transaction_record_kafka_to_clickhouse b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_transaction_record_kafka_to_clickhouse
new file mode 100644
index 0000000..21af021
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_transaction_record_kafka_to_clickhouse
@@ -0,0 +1,159 @@
+sources:
+  kafka_source:
+    type : kafka
+    # fields: # [array of object] Field List, if not set, all fields(Map<String, Object>) will be output.
+    # watermark_timestamp: common_recv_time # [string] Watermark Field Name
+    # watermark_timestamp_unit: ms # [string] Watermark Unit, default is ms
+    # watermark_lag: 60 # [number] Watermark Lag, default is 60
+    properties:
+      topic: TRANSACTION-RECORD
+      kafka.bootstrap.servers: {{ kafka_sink_servers }}
+      kafka.session.timeout.ms: 60000
+      kafka.max.poll.records: 3000
+      kafka.max.partition.fetch.bytes: 31457280
+      kafka.security.protocol: SASL_PLAINTEXT
+      kafka.ssl.keystore.location:
+      kafka.ssl.keystore.password:
+      kafka.ssl.truststore.location:
+      kafka.ssl.truststore.password:
+      kafka.ssl.key.password:
+      kafka.sasl.mechanism: PLAIN
+      kafka.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019";
+      kafka.buffer.memory:
+      kafka.group.id: etl_transaction_record_kafka_to_clickhouse-20231221
+      kafka.auto.offset.reset: latest
+      kafka.max.request.size:
+      kafka.compression.type: none
+      format: json
+
+processing_pipelines:
+  etl_processor: # [object] Processing Pipeline
+    type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl
+    remove_fields:
+    output_fields:
+    properties:
+      key: value
+    functions: # [array of object] Function List
+
+      - function: ASN_LOOKUP
+        lookup_fields: [ server_ip ]
+        output_fields: [ server_asn ]
+        parameters:
+          option: IP_TO_ASN
+          kb_name: tsg_ip_asn
+
+      - function: ASN_LOOKUP
+        lookup_fields: [ client_ip ]
+        output_fields: [ client_asn ]
+        parameters:
+          option: IP_TO_ASN
+          kb_name: tsg_ip_asn
+
+      - function: SNOWFLAKE_ID
+        lookup_fields: [ '' ]
+        output_fields: [ log_id ]
+        parameters:
+          data_center_id_num: 1
+
+      - function: JSON_EXTRACT
+        lookup_fields: [ device_tag ]
+        output_fields: [ data_center ]
+        filter:
+        parameters:
+          value_expression: $.tags[?(@.tag=='data_center')][0].value
+
+      - function: JSON_EXTRACT
+        lookup_fields: [ device_tag ]
+        output_fields: [ device_group ]
+        filter:
+        parameters:
+          value_expression: $.tags[?(@.tag=='device_group')][0].value
+
+      - function: CURRENT_UNIX_TIMESTAMP
+        output_fields: [ processing_time ]
+        parameters:
+          precision: seconds
+
+      - function: UNIX_TIMESTAMP_CONVERTER
+        lookup_fields: [ __timestamp ]
+        output_fields: [ recv_time ]
+        parameters:
+          precision: seconds
+
+      - function: EVAL
+        output_fields: [ ingestion_time ]
+        parameters:
+          value_expression: 'recv_time'
+
+      - function: DOMAIN
+        lookup_fields: [ http_host,ssl_sni,dtls_sni,quic_sni ]
+        output_fields: [ server_domain ]
+        parameters:
+          option: FIRST_SIGNIFICANT_SUBDOMAIN
+
+      - function: BASE64_DECODE_TO_STRING
+        output_fields: [mail_subject]
+        parameters:
+          value_field: mail_subject
+          charset_field: mail_subject_charset
+
+      - function: BASE64_DECODE_TO_STRING
+        output_fields: [mail_attachment_name]
+        parameters:
+          value_field: mail_attachment_name
+          charset_field: mail_attachment_name_charset
+
+      - function: PATH_COMBINE
+        lookup_fields: [ rtp_pcap_path ]
+        output_fields: [ rtp_pcap_path ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, rtp_pcap_path ]
+
+      - function: PATH_COMBINE
+        lookup_fields: [ http_request_body ]
+        output_fields: [ http_request_body ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, http_request_body ]
+
+      - function: PATH_COMBINE
+        lookup_fields: [ http_response_body ]
+        output_fields: [ http_response_body ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, http_response_body ]
+
+      - function: PATH_COMBINE
+        lookup_fields: [ mail_eml_file ]
+        output_fields: [ mail_eml_file ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, mail_eml_file ]
+
+      - function: PATH_COMBINE
+        lookup_fields: [ packet_capture_file ]
+        output_fields: [ packet_capture_file ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, packet_capture_file ]
+
+sinks:
+  clickhouse_sink:
+    type: clickhouse
+    properties:
+      host: {{ clickhouse_servers }}
+      table: tsg_galaxy_v3.transaction_record_local
+      batch.size: 100000
+      batch.interval: 30s
+      connection.user: default
+      connection.password: galaxy2019
+
+
+application:
+
+  env: # [object] Environment Variables
+    name: etl_transaction_record_kafka_to_clickhouse # [string] Job Name
+    pipeline:
+      object-reuse: true # [boolean] Object Reuse, default is false
+  topology:
+    - name: kafka_source
+      downstream: [etl_processor]
+    - name: etl_processor
+      downstream: [clickhouse_sink]
+    - name: clickhouse_sink
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_voip_record_kafka_to_clickhouse b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_voip_record_kafka_to_clickhouse
new file mode 100644
index 0000000..effb6db
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/config/etl_voip_record_kafka_to_clickhouse
@@ -0,0 +1,161 @@
+sources:
+  kafka_source:
+    type : kafka
+    # fields: # [array of object] Field List, if not set, all fields(Map<String, Object>) will be output.
+    # watermark_timestamp: common_recv_time # [string] Watermark Field Name
+    # watermark_timestamp_unit: ms # [string] Watermark Unit, default is ms
+    # watermark_lag: 60 # [number] Watermark Lag, default is 60
+    properties:
+      topic: VOIP-CONVERSATION-RECORD
+      kafka.bootstrap.servers: {{ kafka_sink_servers }}
+      kafka.session.timeout.ms: 60000
+      kafka.max.poll.records: 3000
+      kafka.max.partition.fetch.bytes: 31457280
+      kafka.security.protocol: SASL_PLAINTEXT
+      kafka.ssl.keystore.location:
+      kafka.ssl.keystore.password:
+      kafka.ssl.truststore.location:
+      kafka.ssl.truststore.password:
+      kafka.ssl.key.password:
+      kafka.sasl.mechanism: PLAIN
+      kafka.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019";
+      kafka.buffer.memory:
+      kafka.group.id: etl_voip_record_kafka_to_clickhouse-20231221
+      kafka.auto.offset.reset: latest
+      kafka.max.request.size:
+      kafka.compression.type: none
+      format: json
+
+processing_pipelines:
+  etl_processor: # [object] Processing Pipeline
+    type: com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl
+    remove_fields:
+    output_fields:
+    properties:
+      key: value
+    functions: # [array of object] Function List
+
+      - function: ASN_LOOKUP
+        lookup_fields: [ server_ip ]
+        output_fields: [ server_asn ]
+        parameters:
+          option: IP_TO_ASN
+          kb_name: tsg_ip_asn
+
+      - function: ASN_LOOKUP
+        lookup_fields: [ client_ip ]
+        output_fields: [ client_asn ]
+        parameters:
+          option: IP_TO_ASN
+          kb_name: tsg_ip_asn
+
+      - function: SNOWFLAKE_ID
+        lookup_fields: [ '' ]
+        output_fields: [ log_id ]
+        parameters:
+          data_center_id_num: 1
+
+      - function: JSON_EXTRACT
+        lookup_fields: [ device_tag ]
+        output_fields: [ data_center ]
+        filter:
+        parameters:
+          value_expression: $.tags[?(@.tag=='data_center')][0].value
+
+      - function: JSON_EXTRACT
+        lookup_fields: [ device_tag ]
+        output_fields: [ device_group ]
+        filter:
+        parameters:
+          value_expression: $.tags[?(@.tag=='device_group')][0].value
+
+      - function: CURRENT_UNIX_TIMESTAMP
+        output_fields: [ processing_time ]
+        parameters:
+          precision: seconds
+
+      - function: UNIX_TIMESTAMP_CONVERTER
+        lookup_fields: [ __timestamp ]
+        output_fields: [ recv_time ]
+        parameters:
+          precision: seconds
+
+      - function: EVAL
+        output_fields: [ ingestion_time ]
+        parameters:
+          value_expression: 'recv_time'
+
+      - function: DOMAIN
+        lookup_fields: [ http_host,ssl_sni,dtls_sni,quic_sni ]
+        output_fields: [ server_domain ]
+        parameters:
+          option: FIRST_SIGNIFICANT_SUBDOMAIN
+
+      - function: BASE64_DECODE_TO_STRING
+        output_fields: [mail_subject]
+        parameters:
+          value_field: mail_subject
+          charset_field: mail_subject_charset
+
+      - function: BASE64_DECODE_TO_STRING
+        output_fields: [mail_attachment_name]
+        parameters:
+          value_field: mail_attachment_name
+          charset_field: mail_attachment_name_charset
+
+      - function: PATH_COMBINE
+        lookup_fields: [ rtp_pcap_path ]
+        output_fields: [ rtp_pcap_path ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, rtp_pcap_path ]
+
+      - function: PATH_COMBINE
+        lookup_fields: [ http_request_body ]
+        output_fields: [ http_request_body ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, http_request_body ]
+
+      - function: PATH_COMBINE
+        lookup_fields: [ http_response_body ]
+        output_fields: [ http_response_body ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, http_response_body ]
+
+      - function: PATH_COMBINE
+        lookup_fields: [ mail_eml_file ]
+        output_fields: [ mail_eml_file ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, mail_eml_file ]
+
+      - function: PATH_COMBINE
+        lookup_fields: [ packet_capture_file ]
+        output_fields: [ packet_capture_file ]
+        parameters:
+          path: [ props.hos.path, props.hos.bucket.name.traffic_file, packet_capture_file ]
+
+sinks:
+  clickhouse_sink:
+    type: clickhouse
+    properties:
+      host: {{ clickhouse_servers }}
+      table: tsg_galaxy_v3.voip_record_local
+      batch.size: 100000
+      batch.interval: 30s
+      connection.user: default
+      connection.password: galaxy2019
+
+
+application:
+
+  env: # [object] Environment Variables
+    name: etl_voip_record_kafka_to_clickhouse # [string] Job Name
+    pipeline:
+      object-reuse: true # [boolean] Object Reuse, default is false
+  topology:
+    - name: kafka_source
+      downstream: [etl_processor]
+    - name: etl_processor
+      downstream: [clickhouse_sink]
+    - name: clickhouse_sink
+
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/dos_event_kafka_to_clickhouse.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/dos_event_kafka_to_clickhouse.sh
new file mode 100644
index 0000000..bd7d35c
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/dos_event_kafka_to_clickhouse.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+export PARALLELISM="{{ flink.dos_event_kafka_to_clickhouse['parallelism'] }}"
+export TASK_MODE="{{ task_mode }}"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+{% for item in flink.dos_event_kafka_to_clickhouse | dict2items %}
+{% if item.key != 'parallelism' %}
+-D{{ item.key }}={{ item.value }}
+{% endif %}
+{% endfor %}
+"
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_proxy_event_kafka_to_clickhouse.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_proxy_event_kafka_to_clickhouse.sh
new file mode 100644
index 0000000..9b067c8
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_proxy_event_kafka_to_clickhouse.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+export PARALLELISM="{{ flink.etl_proxy_event_kafka_to_clickhouse['parallelism'] }}"
+
+export TASK_MODE="{{ task_mode }}"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+{% for item in flink.etl_proxy_event_kafka_to_clickhouse | dict2items %}
+{% if item.key != 'parallelism' %}
+-D{{ item.key }}={{ item.value }}
+{% endif %}
+{% endfor %}
+"
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_session_record_kafka_to_clickhouse.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_session_record_kafka_to_clickhouse.sh
new file mode 100644
index 0000000..0f46e7f
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_session_record_kafka_to_clickhouse.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+export PARALLELISM="{{ flink.etl_session_record_kafka_to_clickhouse['parallelism'] }}"
+
+export TASK_MODE="{{ task_mode }}"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+{% for item in flink.etl_session_record_kafka_to_clickhouse | dict2items %}
+{% if item.key != 'parallelism' %}
+-D{{ item.key }}={{ item.value }}
+{% endif %}
+{% endfor %}
+"
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_transaction_record_kafka_to_clickhouse.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_transaction_record_kafka_to_clickhouse.sh
new file mode 100644
index 0000000..bfe455a
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_transaction_record_kafka_to_clickhouse.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+export PARALLELISM="{{ flink.etl_transaction_record_kafka_to_clickhouse['parallelism'] }}"
+
+export TASK_MODE="{{ task_mode }}"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+{% for item in flink.etl_transaction_record_kafka_to_clickhouse | dict2items %}
+{% if item.key != 'parallelism' %}
+-D{{ item.key }}={{ item.value }}
+{% endif %}
+{% endfor %}
+"
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_voip_record_kafka_to_clickhouse.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_voip_record_kafka_to_clickhouse.sh
new file mode 100644
index 0000000..d7815b2
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/env/etl_voip_record_kafka_to_clickhouse.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+export PARALLELISM="{{ flink.etl_voip_record_kafka_to_clickhouse['parallelism'] }}"
+
+export TASK_MODE="{{ task_mode }}"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+{% for item in flink.etl_voip_record_kafka_to_clickhouse | dict2items %}
+{% if item.key != 'parallelism' %}
+-D{{ item.key }}={{ item.value }}
+{% endif %}
+{% endfor %}
+"
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/start.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/start.sh
new file mode 100644
index 0000000..e119510
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/start.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+source /etc/profile
+
+# 任务jar所在目录
+BASE_DIR="$(dirname "$(pwd)")"
+
+SESSION_CLUSTER="Flink session cluster"
+
+ENV_DIR=$BASE_DIR/env/${TASK_NAME}.sh
+
+CONFIG_PATH=${1:-$BASE_DIR/config/}
+
+for file in $(ls "${CONFIG_PATH}"); do
+  if [ -f "$CONFIG_PATH/$file" ]; then
+    TASK_NAME=${file}
+    if [ -f "$BASE_DIR/env/${TASK_NAME}.sh" ]; then
+      # Promote all variable declarations to environment (exported) variables
+      set -a
+      . "$BASE_DIR/env/${TASK_NAME}.sh"
+      set +a
+    else
+      exit 1
+  fi
+    
+  case ${TASK_MODE} in
+    yarn-per-job)
+      HAS_TASK=$(yarn application -list | grep "$TASK_NAME" | wc -l)
+      if [ ${HAS_TASK} -eq "0" ]; then
+        $GROOT_HOME/bin/start.sh --target $TASK_MODE ${FLINK_JOB_OPTS} \
+        -Dyarn.application.name=$TASK_NAME \
+        -Dmetrics.reporter.promgateway.jobName=$TASK_NAME \
+        -p $PARALLELISM -d -c $CONFIG_PATH/$TASK_NAME
+        
+        echo -e "\033[32;1m ${TASK_NAME} job started. \033[0m"
+      else
+      	echo -e "\033[31;1m ${TASK_NAME} job started,Do not submit tasks repeatedly! \033[0m"
+      fi
+      ;;
+    yarn-session)
+      HAS_SESSION=$(yarn application -list | grep "${SESSION_CLUSTER}" | wc -l)
+      if [ ${HAS_SESSION} -ne '0' ]; then
+      	SESSION_ID=$(yarn application -list | grep "${SESSION_CLUSTER}" | awk '{print $1}')
+      	HAS_TASK=$(flink list -t yarn-session -Dyarn.application.id="${SESSION_ID}" | grep "${TASK_NAME}" | grep -v flink | wc -l)
+      	if [ -n "${SESSION_ID}" ] && [ ${HAS_TASK} -eq "0" ]; then
+      	  $GROOT_HOME/bin/start.sh --target $TASK_MODE -Dyarn.application.id=${SESSION_ID} \
+      	  -p ${PARALLELISM} -d -c ${CONFIG_PATH}/${TASK_NAME} -n ${TASK_NAME}
+          
+      	  echo -e "\033[32;1m ${TASK_NAME} job started. \033[0m"
+      	else
+      	  echo -e "\033[31;1m ${TASK_NAME} job started,Do not submit tasks repeatedly! \033[0m"
+      	fi
+      else
+      	echo -e "\033[31;1m Check whether the $SESSION_CLUSTER container is not found in the Yarn cluster. \033[0m"
+      fi
+      ;;
+    *)
+      echo "Invalid DEPLOYMENT_MODE. Choose from: standalone, standalone-docker, yarn-per-job, yarn-session"
+      exit 1
+      ;;
+    esac
+  fi
+done
+
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/stop.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/stop.sh
new file mode 100644
index 0000000..434429c
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/groot-stream/stop.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# flink任务停止脚本
+source /etc/profile
+
+# yarn-per-job, yarn-session
+
+SESSION_CLUSTER="Flink session cluster"
+
+CONFIG_PATH=${1:-$BASE_DIR/config/}
+
+function stopJob() {
+  TASK_NAME=$1
+  
+  case $TASK_MODE in
+  yarn-per-job)
+    HAS_TASK=$(yarn application -list | grep "$TASK_NAME" | wc -l)
+    if [ $HAS_TASK -ne "0" ]; then
+      SESSION_ID=$(yarn application -list | grep "$TASK_NAME" | awk '{print $1}')
+      yarn application -kill $SESSION_ID
+    fi
+    ;;
+  yarn-session)
+    SESSION_ID=$(yarn application -list | grep "$SESSION_CLUSTER" | awk '{print $1}')
+    HAS_TASK=$(flink list -t yarn-session -Dyarn.application.id="$SESSION_ID" | grep "$TASK_NAME" | grep -v flink | wc -l)
+    if [ -n $SESSION_ID ] && [ $HAS_TASK -ne "0" ]; then
+      JOB_ID=$(flink list -t yarn-session -Dyarn.application.id="$SESSION_ID" | grep -v flink | grep "$TASK_NAME" | awk '{print $4}')
+      flink cancel -t yarn-session -Dyarn.application.id=$SESSION_ID $JOB_ID
+    fi
+    ;;
+  *)
+    echo "Invalid DEPLOYMENT_MODE. Choose from: standalone, yarn-per-job, yarn-session"
+    exit 1
+    ;;
+  esac
+}
+
+#如果是文件则停止一个任务
+if [ -f "$1" ]; then
+  stopJob $1
+fi
+
+#如果是文件夹则遍历停止
+if [ -d "$1" ]; then
+  for file in $(ls "$1"); do
+    stopJob $file
+  done
+fi
+
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/sip-rtp-correlation/config/correlation_sip_rtp_session.j2 b/tsg-olap/parcels/roles/initialization/yarn/templates/sip-rtp-correlation/config/correlation_sip_rtp_session.j2
new file mode 100644
index 0000000..0007cd9
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/sip-rtp-correlation/config/correlation_sip_rtp_session.j2
@@ -0,0 +1,46 @@
+#Source configurations
+source.kafka.topic=VOIP-RECORD
+
+source.kafka.props.bootstrap.servers={{ kafka_source_servers }}
+
+source.kafka.props.group.id=sip-rtp-correlation-20230821-1
+
+source.kafka.props.security.protocol=SASL_PLAINTEXT
+
+source.kafka.props.sasl.mechanism=PLAIN
+
+source.kafka.props.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019";
+
+
+#Sink configurations
+sink.kafka.topic=VOIP-CONVERSATION-RECORD
+
+sink.kafka.props.bootstrap.servers={{ kafka_sink_servers }}
+
+sink.kafka.props.security.protocol=SASL_PLAINTEXT
+
+sink.kafka.props.sasl.mechanism=PLAIN
+
+sink.kafka.props.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019";
+
+
+#Correlation configurations
+sip.state.clear.interval.minutes=1
+
+rtp.state.clear.interval.minutes=5
+
+#Error log configurations
+error.records.output.enable=true
+
+error.sink.kafka.topic=VOIP-CONVERSATION-RECORD
+
+error.sink.kafka.props.bootstrap.servers=192.168.45.102:9094
+
+error.sink.kafka.props.security.protocol=SASL_PLAINTEXT
+
+error.sink.kafka.props.sasl.mechanism=PLAIN
+
+error.sink.kafka.props.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019";
+
+include.intranet.ip=true
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/sip-rtp-correlation/env/correlation_sip_rtp_session.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/sip-rtp-correlation/env/correlation_sip_rtp_session.sh
new file mode 100644
index 0000000..ea91f63
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/sip-rtp-correlation/env/correlation_sip_rtp_session.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+export MAIN_CLASS="com.zdjizhi.flink.voip.CorrelateApp"
+
+export PARALLELISM="{{ flink.correlation_sip_rtp_session['parallelism'] }}"
+export TASK_MODE="{{ task_mode }}"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+{% for item in flink.correlation_sip_rtp_session | dict2items %}
+{% if item.key != 'parallelism' %}
+-D{{ item.key }}={{ item.value }}
+{% endif %}
+{% endfor %}
+"
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/start.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/start.sh
new file mode 100644
index 0000000..44b015c
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/start.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+source /etc/profile
+
+# 任务jar所在目录
+BASE_DIR="$(dirname "$(pwd)")"
+
+SESSION_CLUSTER="Flink session cluster"
+
+#Check the number of jar
+if [ `ls ${BASE_DIR}/lib | grep jar | wc -l` -eq '1' ];then
+  JAR_NAME=`ls ${BASE_DIR}/lib | grep jar`
+else
+  echo -e "\033[31;1m There are too many jars in the current directory. Keep the latest files! \033[0m"
+  exit 1
+fi
+
+JAR_DIR=${BASE_DIR}/lib/${JAR_NAME}
+
+CONFIG_PATH=${1:-$BASE_DIR/config/}
+
+for file in $(ls "${CONFIG_PATH}"); do
+if [ -f "$CONFIG_PATH/$file" ]; then
+  TASK_NAME=${file}
+  #Load job env
+  if [ -f "$BASE_DIR/env/${TASK_NAME}.sh" ]; then
+    # Promote all variable declarations to environment (exported) variables
+    set -a
+    . "$BASE_DIR/env/${TASK_NAME}.sh"
+    set +a
+  else
+   exit 1
+  fi
+  
+  case ${TASK_MODE} in
+    yarn-per-job)
+      HAS_TASK=$(yarn application -list | grep "$TASK_NAME" | wc -l)
+      if [ ${HAS_TASK} -eq "0" ]; then
+        flink run -t yarn-per-job ${FLINK_JOB_OPTS} -Dyarn.application.name=${TASK_NAME} -Dmetrics.reporter.promgateway.jobName=${TASK_NAME} \
+		  -p ${PARALLELISM} -d -c ${MAIN_CLASS} ${JAR_DIR} ${CONFIG_PATH}/${TASK_NAME}
+        echo -e "\033[32;1m ${TASK_NAME} job started. \033[0m"
+      else
+        echo -e "\033[31;1m ${TASK_NAME} job started,Do not submit tasks repeatedly! \033[0m"
+      fi
+      ;;
+    yarn-session)
+      HAS_SESSION=$(yarn application -list | grep "${SESSION_CLUSTER}" | wc -l)
+      if [ ${HAS_SESSION} -ne '0' ]; then
+          SESSION_ID=$(yarn application -list | grep "${SESSION_CLUSTER}" | awk '{print $1}')
+          HAS_TASK=$(flink list -t yarn-session -Dyarn.application.id="${SESSION_ID}" | grep "${TASK_NAME}" | grep -v flink | wc -l)
+          if [ -n "${SESSION_ID}" ] && [ ${HAS_TASK} -eq "0" ]; then
+              flink run -t yarn-session -Dyarn.application.id=${SESSION_ID} \
+                -p ${PARALLELISM} -d -c ${MAIN_CLASS} ${JAR_DIR} ${CONFIG_PATH}/${TASK_NAME}
+              echo -e "\033[32;1m ${TASK_NAME} job started. \033[0m"
+          else
+              echo -e "\033[31;1m ${TASK_NAME} job started,Do not submit tasks repeatedly! \033[0m"
+          fi
+      else
+          echo -e "\033[31;1m Check whether the $SESSION_CLUSTER container is not found in the Yarn cluster. \033[0m"
+      fi
+      ;;
+    *)
+      echo "Invalid DEPLOYMENT_MODE. Choose from: standalone, standalone-docker, yarn-per-job, yarn-session"
+      exit 1
+      ;;
+  esac
+fi
+done
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/stop.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/stop.sh
new file mode 100644
index 0000000..86492f7
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/stop.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# flink任务停止脚本
+source /etc/profile
+
+BASE_DIR="$(dirname "$(pwd)")"
+SESSION_CLUSTER="Flink session cluster"
+
+function stopJob(){
+TASK_NAME=$1
+  if [ -f "$BASE_DIR/env/${TASK_NAME}.sh" ]; then
+    # Promote all variable declarations to environment (exported) variables
+    set -a
+    . "$BASE_DIR/env/${TASK_NAME}.sh"
+    set +a
+  else
+   exit 1
+  fi
+
+case ${TASK_MODE} in
+  yarn-per-job)
+    HAS_TASK=$(yarn application -list | grep "${TASK_NAME}" | wc -l)
+    if [ ${HAS_TASK} -ne "0" ]; then
+        SESSION_ID=$(yarn application -list | grep "${TASK_NAME}" | awk '{print $1}')
+        yarn application -kill ${SESSION_ID}
+    fi
+    ;;
+  yarn-session)
+    SESSION_ID=$(yarn application -list | grep "$SESSION_CLUSTER" | awk '{print $1}')
+    HAS_TASK=$(flink list -t yarn-session -Dyarn.application.id="${SESSION_ID}" | grep "${TASK_NAME}" | grep -v flink | wc -l)
+    if [ -n ${SESSION_ID} ] && [ ${HAS_TASK} -ne "0" ]; then
+        JOB_ID=$(flink list -t yarn-session -Dyarn.application.id="${SESSION_ID}" | grep -v flink | grep "${TASK_NAME}" | awk '{print $4}')
+        flink cancel -t yarn-session -Dyarn.application.id=${SESSION_ID} ${JOB_ID}
+    fi
+    ;;
+  *)
+    echo "Invalid DEPLOYMENT_MODE. Choose from: standalone, yarn-per-job, yarn-session"
+    exit 1
+    ;;
+esac
+}
+
+CONFIG_PATH=${1:-$BASE_DIR/config}
+
+#如果是文件则停止一个任务
+if [ -f "$CONFIG_PATH" ]; then
+  stopJob $CONFIG_PATH
+fi
+
+#如果是文件夹则遍历停止
+if [ -d "$CONFIG_PATH" ];then
+  for file in $(ls "$CONFIG_PATH"); do
+    stopJob $file
+  done
+fi
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/topn-metrics-job/config/agg_session_record_topn.j2 b/tsg-olap/parcels/roles/initialization/yarn/templates/topn-metrics-job/config/agg_session_record_topn.j2
new file mode 100644
index 0000000..df040ee
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/topn-metrics-job/config/agg_session_record_topn.j2
@@ -0,0 +1,69 @@
+#--------------------------------Kafka消费组信息------------------------------#
+#kafka的地址信息
+kafka.consumer.broker={{ kafka_source_servers }}
+
+#kafka 接收数据topic
+kafka.consumer.topic=SESSION-RECORD
+
+#消费组名称
+kafka.consumer.group.id=topn-metrics-job-20230501
+
+#--------------------------------Kafka生产者信息------------------------------#
+#kafka的地址信息
+kafka.producer.broker={{ kafka_sink_servers }}
+
+#数据输出topic
+kafka.producer.topic=TRAFFIC-TOP-METRIC
+
+#--------------------------------topology配置------------------------------#
+#任务名称
+job.name=agg_session_record_topn
+
+#任务并行度
+task.parallelism={{ flink.agg_session_record_topn['parallelism'] }}
+
+#source并行度
+kafka.consumer.parallelism={{ flink.agg_session_record_topn['parallelism'] }}
+
+#二次排序并行度
+orderby.parallelism={{ flink.agg_session_record_topn['parallelism'] }}
+
+#入库并行度，通常等于orderby.parallelism
+sink.parallelism={{ flink.agg_session_record_topn['parallelism'] }}
+
+#top结果限制
+top.limit=1000
+
+#窗口延迟等待时间单位秒
+watermark.time=90
+
+#滑动窗口总时间单位分钟
+window.time.minute=1
+
+#--------------------------------Kafka Consumer Config------------------------------#
+#kafka消费者是否开启安全验证 0不开启 1SSL 2 SASL
+kafka.consumer.security=2
+
+#kafka SASL验证用户名
+kafka.consumer.user=admin
+
+#kafka SASL及SSL验证密码
+kafka.consumer.pin=galaxy2019
+
+#1SSL需要
+tools.consumer.library={{ deploy_dir }}/topology/data/
+
+#--------------------------------Kafka Producer Config------------------------------#
+#kafka消费者是否开启安全验证 0不开启 1SSL 2 SASL
+kafka.producer.security=2
+
+#kafka SASL验证用户名
+kafka.producer.user=admin
+
+#kafka SASL及SSL验证密码
+kafka.producer.pin=galaxy2019
+
+#1SSL需要
+tools.producer.library={{ deploy_dir }}/topology/data/
+
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/topn-metrics-job/env/agg_session_record_topn.sh b/tsg-olap/parcels/roles/initialization/yarn/templates/topn-metrics-job/env/agg_session_record_topn.sh
new file mode 100644
index 0000000..939c4e0
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/topn-metrics-job/env/agg_session_record_topn.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+export MAIN_CLASS="com.galaxy.tsg.Toptask"
+
+export PARALLELISM="{{ flink.agg_session_record_topn['parallelism'] }}"
+export TASK_MODE="{{ task_mode }}"
+
+export FLINK_JOB_OPTS="
+-Djobmanager.memory.process.size=1024m
+-Dtaskmanager.memory.process.size={{ flink.agg_session_record_topn['taskmanager.memory.process.size'] }}
+-Dtaskmanager.numberOfTaskSlots={{ flink.agg_session_record_topn['taskmanager.numberOfTaskSlots'] }}
+-Dtaskmanager.memory.framework.off-heap.size={{ flink.agg_session_record_topn['taskmanager.memory.framework.off-heap.size'] }}
+"
diff --git a/tsg-olap/parcels/roles/initialization/yarn/templates/upload_knowlodge.sh.j2 b/tsg-olap/parcels/roles/initialization/yarn/templates/upload_knowlodge.sh.j2
new file mode 100644
index 0000000..9049910
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/templates/upload_knowlodge.sh.j2
@@ -0,0 +1,6 @@
+curl --location --request POST 'http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/v1/knowledge_base' --form 'file=@"{{ deploy_dir }}/flink/topology/data/asn_builtin.mmdb"' --form 'kb_id=f9f6bc91-2142-4673-8249-e097c00fe1ea' --form 'name=asn_builtin' --form 'format=mmdb' --form 'is_valid=1'
+
+curl --location --request POST 'http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/v1/knowledge_base' --form 'file=@"{{ deploy_dir }}/flink/topology/data/ip_builtin.mmdb"' --form 'kb_id=64af7077-eb9b-4b8f-80cf-2ceebc89bea9' --form 'name=ip_builtin' --form 'format=mmdb' --form 'is_valid=1'
+
+curl --location --request POST 'http://{{ vrrp_instance.default.virtual_ipaddress }}:9999/v1/knowledge_base' --form 'file=@"{{ deploy_dir }}/flink/topology/data/ip_user_defined.mmdb"' --form 'kb_id=004390bc-3135-4a6f-a492-3662ecb9e289' --form 'name=ip_user_defined' --form 'format=mmdb' --form 'is_valid=1'
+
diff --git a/tsg-olap/parcels/roles/initialization/yarn/vars/main.yml b/tsg-olap/parcels/roles/initialization/yarn/vars/main.yml
new file mode 100644
index 0000000..5f59e9a
--- /dev/null
+++ b/tsg-olap/parcels/roles/initialization/yarn/vars/main.yml
@@ -0,0 +1,21 @@
+task_mode: "{{ 'yarn-per-job' if groups['yarn'] |length > 1 else 'yarn-session' }}"
+
+kafka_source_servers: "{% for host in groups['kafka'] %}{{ host }}:9094{% if not loop.last %},{% endif %}{% endfor %}"
+
+kafka_sink_servers: "{{ kafka_source_servers }}"
+
+zookeeper_servers: "{% for host in groups['kafka'] %}{{ host }}:2181{% if not loop.last %},{% endif %}{% endfor %}"
+
+clickhouse_servers: "{% if groups['clickhouse'] | length < 3 %}{{ groups['clickhouse'][0] }}:9001{% else %}{% for host in groups['clickhouse'][2:] %}{{ host }}:9001{% if not loop.last %},{% endif %}{% endfor %}{% endif %}"
+
+#groot版本
+groot_stream_version: groot-stream-1.0
+
+#Jdk版本
+java_version: 1.8.0_73
+
+
+backup_path: "{{ deploy_dir }}/backup/tsg-olap/{{ old_version }}/flink"
+
+backup_items:
+  - topology