Merge remote-tracking branch 'origin/develop' into develop

3 files changed, 25 insertions, 752 deletions

diff --git a/galaxy-data-engine/schema/clickhouse/connection_record_log.avsc b/galaxy-data-engine/schema/clickhouse/connection_record_log.avsc
index 0956ae5..3687b50 100644
--- a/galaxy-data-engine/schema/clickhouse/connection_record_log.avsc
+++ b/galaxy-data-engine/schema/clickhouse/connection_record_log.avsc
@@ -2,7 +2,7 @@
   "namespace": "tsg_galaxy_v3",
   "type": "record",
   "name": "connection_record_log",
-  "doc": "{\"primary_key\":\"common_log_id\",\"partition_key\":\"common_recv_time\",\"index_table\":\"connection_record_log_common_client_ip,connection_record_log_common_server_ip,connection_record_log_common_subscriber_id,connection_record_log_http_domain\",\"schema_query\":{\"dimensions\":[\"common_server_ip\",\"common_client_ip\",\"common_sled_ip\",\"common_server_location\",\"common_subscriber_id\",\"common_server_port\",\"http_domain\"],\"metrics\":[\"common_server_ip\",\"common_client_ip\",\"common_subscriber_id\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\"],\"filters\":[\"common_address_type\",\"common_server_ip\",\"common_client_ip\",\"common_server_port\",\"common_server_location\",\"common_subscriber_id\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_l4_protocol\",\"http_domain\",\"ssl_sni\"]},\"schema_type\":{\"BASE\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\"]},\"HTTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"http_url\",\"http_host\",\"http_domain\",\"http_request_line\",\"http_response_line\",\"http_request_header\",\"http_response_header\",\"http_request_body\",\"http_response_body\",\"http_request_body_key\",\"http_response_body_key\",\"http_proxy_flag\",\"http_sequence\",\"http_snapshot\",\"http_cookie\",\"http_referer\",\"http_user_agent\",\"http_content_length\",\"http_content_type\",\"http_set_cookie\",\"http_version\",\"http_response_lantency_ms\",\"http_session_duration_ms\",\"http_action_file_size\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"http_url\",\"common_server_port\"]},\"MAIL\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"mail_protocol_type\",\"mail_account\",\"mail_from_cmd\",\"mail_to_cmd\",\"mail_from\",\"mail_to\",\"mail_cc\",\"mail_bcc\",\"mail_subject\",\"mail_subject_charset\",\"mail_content\",\"mail_content_charset\",\"mail_attachment_name\",\"mail_attachment_name_charset\",\"mail_attachment_content\",\"mail_eml_file\",\"mail_snapshot\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"mail_from\",\"mail_to\",\"mail_subject\"]},\"DNS\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"dns_message_id\",\"dns_qr\",\"dns_opcode\",\"dns_aa\",\"dns_tc\",\"dns_rd\",\"dns_ra\",\"dns_rcode\",\"dns_qdcount\",\"dns_ancount\",\"dns_nscount\",\"dns_arcount\",\"dns_qname\",\"dns_qtype\",\"dns_qclass\",\"dns_cname\",\"dns_sub\",\"dns_rr\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_client_ip\",\"dns_qr\",\"dns_qname\",\"dns_qtype\"]},\"SSL\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"ssl_sni\",\"ssl_san\",\"ssl_cn\",\"ssl_pinningst\",\"ssl_intercept_state\",\"ssl_server_side_latency\",\"ssl_client_side_latency\",\"ssl_server_side_version\",\"ssl_client_side_version\",\"ssl_cert_verify\",\"ssl_error\",\"ssl_con_latency_ms\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"ssl_sni\",\"common_server_ip\",\"common_server_port\"]},\"QUIC\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"quic_version\",\"quic_sni\",\"quic_user_agent\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"quic_sni\",\"common_server_ip\",\"common_server_port\"]},\"FTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"ftp_account\",\"ftp_url\",\"ftp_content\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"ftp_url\",\"common_server_ip\",\"common_server_port\"]},\"BGP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"bgp_type\",\"bgp_as_num\",\"bgp_route\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"bgp_type\",\"bgp_as_num\",\"common_server_ip\",\"common_server_port\"]},\"VOIP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"voip_calling_account\",\"voip_called_account\",\"voip_calling_number\",\"voip_called_number\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"voip_calling_account\",\"voip_called_account\",\"common_server_ip\",\"common_server_port\"]}},\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\",\"common_schema_type\"]}",
+  "doc": "{\"primary_key\":\"common_log_id\",\"partition_key\":\"common_recv_time\",\"index_table\":\"connection_record_log_common_client_ip,connection_record_log_common_server_ip,connection_record_log_common_subscriber_id,connection_record_log_http_domain\",\"schema_query\":{\"dimensions\":[\"common_server_ip\",\"common_client_ip\",\"common_sled_ip\",\"common_server_location\",\"common_subscriber_id\",\"common_server_port\",\"http_domain\"],\"metrics\":[\"common_server_ip\",\"common_client_ip\",\"common_subscriber_id\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_sessions\"],\"filters\":[\"common_address_type\",\"common_server_ip\",\"common_client_ip\",\"common_server_port\",\"common_server_location\",\"common_subscriber_id\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_l4_protocol\",\"http_domain\",\"ssl_sni\"]},\"schema_type\":{\"BASE\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\"]},\"HTTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"http_url\",\"http_host\",\"http_domain\",\"http_request_line\",\"http_response_line\",\"http_request_header\",\"http_response_header\",\"http_request_body\",\"http_response_body\",\"http_request_body_key\",\"http_response_body_key\",\"http_proxy_flag\",\"http_sequence\",\"http_snapshot\",\"http_cookie\",\"http_referer\",\"http_user_agent\",\"http_content_length\",\"http_content_type\",\"http_set_cookie\",\"http_version\",\"http_response_lantency_ms\",\"http_session_duration_ms\",\"http_action_file_size\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"http_url\",\"common_server_port\"]},\"MAIL\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"mail_protocol_type\",\"mail_account\",\"mail_from_cmd\",\"mail_to_cmd\",\"mail_from\",\"mail_to\",\"mail_cc\",\"mail_bcc\",\"mail_subject\",\"mail_subject_charset\",\"mail_content\",\"mail_content_charset\",\"mail_attachment_name\",\"mail_attachment_name_charset\",\"mail_attachment_content\",\"mail_eml_file\",\"mail_snapshot\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"mail_from\",\"mail_to\",\"mail_subject\"]},\"DNS\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"dns_message_id\",\"dns_qr\",\"dns_opcode\",\"dns_aa\",\"dns_tc\",\"dns_rd\",\"dns_ra\",\"dns_rcode\",\"dns_qdcount\",\"dns_ancount\",\"dns_nscount\",\"dns_arcount\",\"dns_qname\",\"dns_qtype\",\"dns_qclass\",\"dns_cname\",\"dns_sub\",\"dns_rr\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_client_ip\",\"dns_qr\",\"dns_qname\",\"dns_qtype\"]},\"SSL\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"ssl_sni\",\"ssl_san\",\"ssl_cn\",\"ssl_pinningst\",\"ssl_intercept_state\",\"ssl_server_side_latency\",\"ssl_client_side_latency\",\"ssl_server_side_version\",\"ssl_client_side_version\",\"ssl_cert_verify\",\"ssl_error\",\"ssl_con_latency_ms\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"ssl_sni\",\"common_server_ip\",\"common_server_port\"]},\"QUIC\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"quic_version\",\"quic_sni\",\"quic_user_agent\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"quic_sni\",\"common_server_ip\",\"common_server_port\"]},\"FTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"ftp_account\",\"ftp_url\",\"ftp_content\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"ftp_url\",\"common_server_ip\",\"common_server_port\"]},\"BGP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"bgp_type\",\"bgp_as_num\",\"bgp_route\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"bgp_type\",\"bgp_as_num\",\"common_server_ip\",\"common_server_port\"]},\"VOIP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"voip_calling_account\",\"voip_called_account\",\"voip_calling_number\",\"voip_called_number\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"voip_calling_account\",\"voip_called_account\",\"common_server_ip\",\"common_server_port\"]}},\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\",\"common_schema_type\"]}",
   "fields": [
     {
       "name": "common_recv_time",
@@ -109,8 +109,7 @@
     {
       "name": "common_sessions",
       "type": "int",
-      "label": "Sessions",
-      "doc": "{\"visibility\":\"hidden\"}"
+      "label": "Sessions"
     },
     {
       "name": "common_c2s_pkt_num",
diff --git a/galaxy-data-engine/schema/clickhouse/connection_sketch.avsc b/galaxy-data-engine/schema/clickhouse/connection_sketch.avsc
deleted file mode 100644
index acc87c8..0000000
--- a/galaxy-data-engine/schema/clickhouse/connection_sketch.avsc
+++ /dev/null
@@ -1,743 +0,0 @@
-{
-  "namespace": "tsg_galaxy_v3",
-  "type": "record",
-  "name": "connection_sketch",
-  "doc": "{\"primary_key\":\"common_log_id\",\"partition_key\":\"common_recv_time\",\"schema_query\":{\"dimensions\":[\"common_server_ip\",\"common_client_ip\",\"common_sled_ip\",\"common_server_location\",\"common_subscriber_id\",\"common_server_port\",\"http_domain\"],\"metrics\":[\"common_server_ip\",\"common_client_ip\",\"common_subscriber_id\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\"],\"filters\":[\"common_address_type\",\"common_server_ip\",\"common_client_ip\",\"common_server_port\",\"common_server_location\",\"common_subscriber_id\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_l4_protocol\",\"http_domain\",\"ssl_sni\"]},\"schema_type\":{\"BASE\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\"]},\"HTTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"http_url\",\"http_host\",\"http_domain\",\"http_request_line\",\"http_response_line\",\"http_request_header\",\"http_response_header\",\"http_request_body\",\"http_response_body\",\"http_request_body_key\",\"http_response_body_key\",\"http_proxy_flag\",\"http_sequence\",\"http_snapshot\",\"http_cookie\",\"http_referer\",\"http_user_agent\",\"http_content_length\",\"http_content_type\",\"http_set_cookie\",\"http_version\",\"http_response_lantency_ms\",\"http_session_duration_ms\",\"http_action_file_size\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"http_url\",\"common_server_port\"]},\"MAIL\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"mail_protocol_type\",\"mail_account\",\"mail_from_cmd\",\"mail_to_cmd\",\"mail_from\",\"mail_to\",\"mail_cc\",\"mail_bcc\",\"mail_subject\",\"mail_subject_charset\",\"mail_content\",\"mail_content_charset\",\"mail_attachment_name\",\"mail_attachment_name_charset\",\"mail_attachment_content\",\"mail_eml_file\",\"mail_snapshot\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"mail_from\",\"mail_to\",\"mail_subject\"]},\"DNS\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"dns_message_id\",\"dns_qr\",\"dns_opcode\",\"dns_aa\",\"dns_tc\",\"dns_rd\",\"dns_ra\",\"dns_rcode\",\"dns_qdcount\",\"dns_ancount\",\"dns_nscount\",\"dns_arcount\",\"dns_qname\",\"dns_qtype\",\"dns_qclass\",\"dns_cname\",\"dns_sub\",\"dns_rr\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_client_ip\",\"dns_qr\",\"dns_qname\",\"dns_qtype\"]},\"SSL\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"ssl_sni\",\"ssl_san\",\"ssl_cn\",\"ssl_pinningst\",\"ssl_intercept_state\",\"ssl_server_side_latency\",\"ssl_client_side_latency\",\"ssl_server_side_version\",\"ssl_client_side_version\",\"ssl_cert_verify\",\"ssl_error\",\"ssl_con_latency_ms\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"ssl_sni\",\"common_server_ip\",\"common_server_port\"]},\"QUIC\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"quic_version\",\"quic_sni\",\"quic_user_agent\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"quic_sni\",\"common_server_ip\",\"common_server_port\"]},\"FTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"ftp_account\",\"ftp_url\",\"ftp_content\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"ftp_url\",\"common_server_ip\",\"common_server_port\"]},\"BGP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"bgp_type\",\"bgp_as_num\",\"bgp_route\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"bgp_type\",\"bgp_as_num\",\"common_server_ip\",\"common_server_port\"]},\"VOIP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"voip_calling_account\",\"voip_called_account\",\"voip_calling_number\",\"voip_called_number\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"voip_calling_account\",\"voip_called_account\",\"common_server_ip\",\"common_server_port\"]}},\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\",\"common_schema_type\"]}",
-  "fields": [
-    {
-      "name": "common_recv_time",
-      "label": "Receive Time",
-      "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"current_timestamp\"}}",
-      "type": "int"
-    },
-    {
-      "name": "common_log_id",
-      "label": "Log ID",
-      "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"snowflake_id\"}}",
-      "type": "long"
-    },
-    {
-      "name": "common_policy_id",
-      "label": "Policy ID",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "int"
-    },
-    {
-      "name": "common_subscriber_id",
-      "label": "Subscriber ID",
-      "doc": "{\"allow_query\":\"true\"}",
-      "type": "string"
-    },
-    {
-      "name": "common_client_ip",
-      "label": "Client IP",
-      "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_ip_detail,geo_asn,radius_match\",\"appendTo\":\"common_client_location,common_client_asn,common_subscriber_id\"}}",
-      "type": "string"
-    },
-    {
-      "name": "common_client_port",
-      "label": "Client Port",
-      "type": "int"
-    },
-    {
-      "name": "common_l4_protocol",
-      "label": "L4 Protocol",
-      "type": "string"
-    },
-    {
-      "name": "common_address_type",
-      "label": "Address Type",
-      "doc": "{\"data\":[{\"code\":\"4\",\"value\":\"ipv4\"},{\"code\":\"6\",\"value\":\"ipv6\"}]}",
-      "type": "int"
-    },
-    {
-      "name": "common_server_ip",
-      "label": "Server IP",
-      "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_ip_country,geo_asn\",\"appendTo\":\"common_server_location,common_server_asn\"}}",
-      "type": "string"
-    },
-    {
-      "name": "common_server_port",
-      "label": "Server Port",
-      "doc": "{\"allow_query\":\"true\"}",
-      "type": "int"
-    },
-    {
-      "name": "common_action",
-      "label": "Action",
-      "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"None\"},{\"code\":\"1\",\"value\":\"Monitor\"},{\"code\":\"2\",\"value\":\"Intercept\"},{\"code\":\"16\",\"value\":\"Deny\"},{\"code\":\"128\",\"value\":\"Allow\"}]}",
-      "type": "int"
-    },
-    {
-      "name": "common_direction",
-      "label": "Direction",
-      "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"outbound\"},{\"code\":\"1\",\"value\":\"inbound\"}]}",
-      "type": "int"
-    },
-    {
-      "name": "common_entrance_id",
-      "label": "Entrance ID",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "int"
-    },
-    {
-      "name": "common_sled_ip",
-      "label": "Sled IP",
-      "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}",
-      "type": "string"
-    },
-    {
-      "name": "common_client_location",
-      "label": "Client Location",
-      "type": "string"
-    },
-    {
-      "name": "common_client_asn",
-      "label": "Client ASN",
-      "type": "string"
-    },
-    {
-      "name": "common_server_location",
-      "label": "Server Location",
-      "type": "string"
-    },
-    {
-      "name": "common_server_asn",
-      "label": "Server ASN",
-      "type": "string"
-    },
-    {
-      "name": "common_sessions",
-      "label": "Sessions",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "int"
-    },
-    {
-      "name": "common_c2s_pkt_num",
-      "label": "Packets Sent",
-      "type": "long"
-    },
-    {
-      "name": "common_s2c_pkt_num",
-      "label": "Packets Received",
-      "type": "long"
-    },
-    {
-      "name": "common_c2s_byte_num",
-      "label": "Bytes Sent",
-      "type": "long"
-    },
-    {
-      "name": "common_s2c_byte_num",
-      "label": "Bytes Received",
-      "type": "long"
-    },
-    {
-      "name": "common_service",
-      "label": "Service",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "int"
-    },
-    {
-      "name": "common_schema_type",
-      "label": "Schema Type",
-      "doc": "{\"data\":[{\"code\":\"BASE\",\"value\":\"BASE\"},{\"code\":\"HTTP\",\"value\":\"HTTP\"},{\"code\":\"SSL\",\"value\":\"SSL\"}],\"allow_query\":\"true\"}",
-      "type": "string"
-    },
-    {
-      "name": "common_user_tags",
-      "label": "User Tags",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "string"
-    },
-    {
-      "name": "common_sub_action",
-      "label": "Sub Action",
-      "type": "string"
-    },
-    {
-      "name": "common_user_region",
-      "label": "User Region",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "string"
-    },
-    {
-      "name": "common_device_id",
-      "label": "Device ID",
-      "type": "string"
-    },
-    {
-      "name": "common_link_id",
-      "label": "Link ID",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "int"
-    },
-    {
-      "name": "common_isp",
-      "label": "ISP",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "string"
-    },
-    {
-      "name": "common_encapsulation",
-      "label": "Encapsulation",
-      "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"Ethernet\"},{\"code\":\"8\",\"value\":\"PPP\"},{\"code\":\"12\",\"value\":\"CiscoHDLC\"}],\"visibility\":\"hidden\"}",
-      "type": "int"
-    },
-    {
-      "name": "common_app_label",
-      "label": "Application Label",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "string"
-    },
-    {
-      "name": "common_protocol_label",
-      "label": "Protocol Label",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "string"
-    },
-    {
-      "name": "common_start_time",
-      "label": "Start Time",
-      "doc": "{\"constraints\":{\"type\":\"timestamp\"}}",
-      "type": "int"
-    },
-    {
-      "name": "common_end_time",
-      "label": "End Time",
-      "doc": "{\"constraints\":{\"type\":\"timestamp\"}}",
-      "type": "int"
-    },
-    {
-      "name": "common_establish_latency_ms",
-      "label": "Establish Latency(ms)",
-      "type": "int"
-    },
-    {
-      "name": "common_con_duration_ms",
-      "label": "Duration(ms)",
-      "type": "int"
-    },
-    {
-      "name": "common_stream_dir",
-      "label": "Stream Direction",
-      "doc": "{\"data\":[{\"code\":\"1\",\"value\":\"c2s\"},{\"code\":\"2\",\"value\":\"s2c\"},{\"code\":\"3\",\"value\":\"double\"}]}",
-      "type": "int"
-    },
-    {
-      "name": "common_address_list",
-      "label": "Address List",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "string"
-    },
-    {
-      "name": "common_has_dup_traffic",
-      "label": "Duplication Traffic",
-      "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"No\"},{\"code\":\"1\",\"value\":\"Yes\"}],\"visibility\":\"hidden\"}",
-      "type": "int"
-    },
-    {
-      "name": "common_stream_error",
-      "label": "Stream Error",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "string"
-    },
-    {
-      "name": "common_stream_trace_id",
-      "label": "Session ID",
-      "doc": "{\"allow_query\":\"true\"}",
-      "type": "long"
-    },
-    {
-      "name": "common_link_info_c2s",
-      "label": "Link Info(c2s)",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "string"
-    },
-    {
-      "name": "common_link_info_s2c",
-      "label": "Link Info(s2c)",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "string"
-    },
-    {
-      "name": "http_url",
-      "label": "Http.URL",
-      "type": "string"
-    },
-    {
-      "name": "http_host",
-      "label": "Http.Host",
-      "doc": "{\"format\":{\"functions\":\"sub_domain\",\"appendTo\":\"http_domain\"}}",
-      "type": "string"
-    },
-    {
-      "name": "http_domain",
-      "label": "Http.Domain",
-      "doc": "{\"allow_query\":\"true\"}",
-      "type": "string"
-    },
-    {
-      "name": "http_request_line",
-      "label": "Http.Request Line",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "string"
-    },
-    {
-      "name": "http_response_line",
-      "label": "Http.Response Line",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "string"
-    },
-    {
-      "name": "http_request_header",
-      "label": "Http.Request Headers",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "string"
-    },
-    {
-      "name": "http_response_header",
-      "label": "Http.Response Headers",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "string"
-    },
-    {
-      "name": "http_request_body",
-      "label": "Http.Request Body",
-      "doc": "{\"constraints\":{\"type\":\"file\"}}",
-      "type": "string"
-    },
-    {
-      "name": "http_response_body",
-      "label": "Http.Response Body",
-      "doc": "{\"constraints\":{\"type\":\"file\"}}",
-      "type": "string"
-    },
-    {
-      "name": "http_request_body_key",
-      "label": "Http.Request Body Key",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "string"
-    },
-    {
-      "name": "http_response_body_key",
-      "label": "Http.Response Body Key",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "string"
-    },
-    {
-      "name": "http_proxy_flag",
-      "label": "Http.Proxy Flag",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "int"
-    },
-    {
-      "name": "http_sequence",
-      "label": "Http.Sequence",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "int"
-    },
-    {
-      "name": "http_snapshot",
-      "label": "Http.Snapshot",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "string"
-    },
-    {
-      "name": "http_cookie",
-      "label": "Http.Cookie",
-      "type": "string"
-    },
-    {
-      "name": "http_referer",
-      "label": "Http.Referer",
-      "type": "string"
-    },
-    {
-      "name": "http_user_agent",
-      "label": "Http.User Agent",
-      "type": "string"
-    },
-    {
-      "name": "http_content_length",
-      "label": "Http.Content Length",
-      "type": "string"
-    },
-    {
-      "name": "http_content_type",
-      "label": "Http.Content Type",
-      "type": "string"
-    },
-    {
-      "name": "http_set_cookie",
-      "label": "Http.Set Cookie",
-      "type": "string"
-    },
-    {
-      "name": "http_version",
-      "label": "Http.Version",
-      "type": "string"
-    },
-    {
-      "name": "http_response_lantency_ms",
-      "label": "Http.Response Latency(ms)",
-      "type": "int"
-    },
-    {
-      "name": "http_session_duration_ms",
-      "label": "Http.Session Duration(ms)",
-      "type": "int"
-    },
-    {
-      "name": "http_action_file_size",
-      "label": "Http.Action File Size",
-      "type": "int"
-    },
-    {
-      "name": "mail_protocol_type",
-      "label": "Mail.Protocol Type",
-      "type": "string"
-    },
-    {
-      "name": "mail_account",
-      "label": "Mail.Account",
-      "type": "string"
-    },
-    {
-      "name": "mail_from_cmd",
-      "label": "Mail.From CMD",
-      "type": "string"
-    },
-    {
-      "name": "mail_to_cmd",
-      "label": "Mail.To CMD",
-      "type": "string"
-    },
-    {
-      "name": "mail_from",
-      "label": "Mail.From",
-      "doc": "{\"constraints\":{\"type\":\"email\"}}",
-      "type": "string"
-    },
-    {
-      "name": "mail_to",
-      "label": "Mail.To",
-      "doc": "{\"constraints\":{\"type\":\"email\"}}",
-      "type": "string"
-    },
-    {
-      "name": "mail_cc",
-      "label": "Mail.CC",
-      "type": "string"
-    },
-    {
-      "name": "mail_bcc",
-      "label": "Mail.BCC",
-      "type": "string"
-    },
-    {
-      "name": "mail_subject",
-      "label": "Mail.Subject",
-      "doc": "{\"format\":{\"functions\":\"decode_of_base64\",\"param\":\"mail_subject_charset\"}}",
-      "type": "string"
-    },
-    {
-      "name": "mail_subject_charset",
-      "label": "Mail.Subject Charset",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "string"
-    },
-    {
-      "name": "mail_content",
-      "label": "Mail.Content",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "string"
-    },
-    {
-      "name": "mail_content_charset",
-      "label": "Mail.Content Charset",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "string"
-    },
-    {
-      "name": "mail_attachment_name",
-      "label": "Mail.Attachment",
-      "doc": "{\"format\":{\"functions\":\"decode_of_base64\",\"param\":\"mail_attachment_name_charset\"}}",
-      "type": "string"
-    },
-    {
-      "name": "mail_attachment_name_charset",
-      "label": "Mail.Attachment Charset",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "string"
-    },
-    {
-      "name": "mail_attachment_content",
-      "label": "Mail.Attachment Content",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "string"
-    },
-    {
-      "name": "mail_eml_file",
-      "label": "Mail.EML File",
-      "doc": "{\"constraints\":{\"type\":\"file\"}}",
-      "type": "string"
-    },
-    {
-      "name": "mail_snapshot",
-      "label": "Mail.Snapshot",
-      "doc": "{\"visibility\":\"hidden\"}",
-      "type": "string"
-    },
-    {
-      "name": "dns_message_id",
-      "label": "Dns.Message ID",
-      "type": "int"
-    },
-    {
-      "name": "dns_qr",
-      "label": "Dns.QR",
-      "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"QUERY\"},{\"code\":\"1\",\"value\":\"REESPONSE\"}]}",
-      "type": "int"
-    },
-    {
-      "name": "dns_opcode",
-      "label": "Dns.OPCODE",
-      "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"QUERY\"},{\"code\":\"1\",\"value\":\"IQUERY\"},{\"code\":\"2\",\"value\":\"STATUS\"},{\"code\":\"5\",\"value\":\"UPDATE\"}]}",
-      "type": "int"
-    },
-    {
-      "name": "dns_aa",
-      "label": "Dns.AA",
-      "type": "int"
-    },
-    {
-      "name": "dns_tc",
-      "label": "Dns.TC",
-      "type": "int"
-    },
-    {
-      "name": "dns_rd",
-      "label": "Dns.RD",
-      "type": "int"
-    },
-    {
-      "name": "dns_ra",
-      "label": "Dns.RA",
-      "type": "int"
-    },
-    {
-      "name": "dns_rcode",
-      "label": "Dns.RCODE",
-      "type": "int"
-    },
-    {
-      "name": "dns_qdcount",
-      "label": "Dns.QDCOUNT",
-      "type": "int"
-    },
-    {
-      "name": "dns_ancount",
-      "label": "Dns.ANCOUNT",
-      "type": "int"
-    },
-    {
-      "name": "dns_nscount",
-      "label": "Dns.NSCOUNT",
-      "type": "int"
-    },
-    {
-      "name": "dns_arcount",
-      "label": "Dns.ARCOUNT",
-      "type": "int"
-    },
-    {
-      "name": "dns_qname",
-      "label": "Dns.QNAME",
-      "type": "string"
-    },
-    {
-      "name": "dns_qtype",
-      "label": "Dns.QTYPE",
-      "doc": "{\"data\":[{\"code\":\"1\",\"value\":\"A\"},{\"code\":\"2\",\"value\":\"NS\"},{\"code\":\"5\",\"value\":\"CNAME\"},{\"code\":\"6\",\"value\":\"SOA\"},{\"code\":\"11\",\"value\":\"WKS\"},{\"code\":\"12\",\"value\":\"PTR\"},{\"code\":\"13\",\"value\":\"HINFO\"},{\"code\":\"11\",\"value\":\"WKS\"},{\"code\":\"15\",\"value\":\"MX\"},{\"code\":\"28\",\"value\":\"AAAA\"}]}",
-      "type": "int"
-    },
-    {
-      "name": "dns_qclass",
-      "label": "Dns.QCLASS",
-      "type": "int"
-    },
-    {
-      "name": "dns_cname",
-      "label": "Dns.CNAME",
-      "type": "string"
-    },
-    {
-      "name": "dns_sub",
-      "label": "Dns.SUB",
-      "doc": "{\"data\":[{\"code\":\"1\",\"value\":\"DNS\"},{\"code\":\"2\",\"value\":\"DNSSEC\"}]}",
-      "type": "int"
-    },
-    {
-      "name": "dns_rr",
-      "label": "Dns.RR",
-      "type": "string"
-    },
-    {
-      "name": "ssl_version",
-      "label": "SSL.Version",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "string"
-    },
-    {
-      "name": "ssl_sni",
-      "label": "SSL.SNI",
-      "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"sub_domain\",\"appendTo\":\"http_domain\"}}",
-      "type": "string"
-    },
-    {
-      "name": "ssl_san",
-      "label": "SSL.SAN",
-      "type": "string"
-    },
-    {
-      "name": "ssl_cn",
-      "label": "SSL.CN",
-      "type": "string"
-    },
-    {
-      "name": "ssl_pinningst",
-      "label": "SSL.Pinning",
-      "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"Not Pinning\"},{\"code\":\"1\",\"value\":\"Pinning\"},{\"code\":\"2\",\"value\":\"Maybe Pinning\"}]}",
-      "type": "int"
-    },
-    {
-      "name": "ssl_intercept_state",
-      "label": "SSL.Intercept State",
-      "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"Passthrough\"},{\"code\":\"1\",\"value\":\"Intercept\"},{\"code\":\"2\",\"value\":\"Shutdown\"}]}",
-      "type": "int"
-    },
-    {
-      "name": "ssl_server_side_latency",
-      "label": "SSL.Server Side Latency(ms)",
-      "type": "int"
-    },
-    {
-      "name": "ssl_client_side_latency",
-      "label": "SSL.Client Side Latency(ms)",
-      "type": "int"
-    },
-    {
-      "name": "ssl_server_side_version",
-      "label": "SSL.Server Side Version",
-      "type": "string"
-    },
-    {
-      "name": "ssl_client_side_version",
-      "label": "SSL.Client Side Version",
-      "type": "string"
-    },
-    {
-      "name": "ssl_cert_verify",
-      "label": "SSL.Certificate Verify",
-      "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"No\"},{\"code\":\"1\",\"value\":\"Yes\"}]}",
-      "type": "int"
-    },
-    {
-      "name": "ssl_error",
-      "label": "SSL.Error",
-      "type": "string"
-    },
-    {
-      "name": "ssl_con_latency_ms",
-      "label": "SSL.Connection Latency(ms)",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "int"
-    },
-    {
-      "name": "quic_version",
-      "label": "Quic.Version",
-      "type": "string"
-    },
-    {
-      "name": "quic_sni",
-      "label": "Quic.SNI",
-      "doc": "{\"format\":{\"functions\":\"sub_domain\",\"appendTo\":\"http_domain\"}}",
-      "type": "string"
-    },
-    {
-      "name": "quic_user_agent",
-      "label": "Quic.User Agent",
-      "type": "string"
-    },
-    {
-      "name": "ftp_account",
-      "label": "Ftp.Account",
-      "type": "string"
-    },
-    {
-      "name": "ftp_url",
-      "label": "Ftp.URL",
-      "type": "string"
-    },
-    {
-      "name": "ftp_content",
-      "label": "Ftp.Content",
-      "type": "string"
-    },
-    {
-      "name": "bgp_type",
-      "label": "BGP.Type",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "int"
-    },
-    {
-      "name": "bgp_as_num",
-      "label": "BGP.AS Number",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "string"
-    },
-    {
-      "name": "bgp_route",
-      "label": "BGP.Route",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "string"
-    },
-    {
-      "name": "voip_calling_account",
-      "label": "Voip.Calling Account",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "string"
-    },
-    {
-      "name": "voip_called_account",
-      "label": "Voip.Called Account",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "string"
-    },
-    {
-      "name": "voip_calling_number",
-      "label": "Voip.Calling Number",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "string"
-    },
-    {
-      "name": "voip_called_number",
-      "label": "Voip.Called Number",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "string"
-    },
-    {
-      "name": "streaming_media_url",
-      "label": "Streaming.Media URL",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "string"
-    },
-    {
-      "name": "streaming_media_protocol",
-      "label": "Streaming.Media Protocol",
-      "doc": "{\"visibility\":\"disabled\"}",
-      "type": "string"
-    }
-  ]
-}
diff --git a/galaxy-data-engine/src/main/java/com/mesalab/engine/component/dialect/ClickHouseDialect.java b/galaxy-data-engine/src/main/java/com/mesalab/engine/component/dialect/ClickHouseDialect.java
index d4e3506..468f347 100644
--- a/galaxy-data-engine/src/main/java/com/mesalab/engine/component/dialect/ClickHouseDialect.java
+++ b/galaxy-data-engine/src/main/java/com/mesalab/engine/component/dialect/ClickHouseDialect.java
@@ -609,12 +609,16 @@ public class ClickHouseDialect extends AbstractDataSourceDialect {
                     if (orderField.startsWith("\"") && orderField.endsWith("\"")) {
                         orderField = orderField.substring(1, orderField.length() - 1);
                     }
-                    String originalField = aliasFields.get(orderField);
-                    if (StringUtil.isNotBlank(originalField) && !isFieldOfSchema(schema, originalField)) {
-                        return false;
-                    }
-                    if (StringUtil.isBlank(originalField) && !isFieldOfSchema(schema, orderField)) {
-                        return false;
+                    String originalField = StringUtil.isBlank(aliasFields.get(orderField)) ? orderField : aliasFields.get(orderField);
+                    Expression expression = parserException(originalField);
+                    if (expression instanceof Function) {
+                        if (!getValidityByFunctionParamsInSchema(schema, (Function) expression)) {
+                            return false;
+                        }
+                    } else {
+                        if (!isFieldOfSchema(schema, originalField)) {
+                            return false;
+                        }
                     }
                 }
                 return true;
@@ -691,6 +695,10 @@ public class ClickHouseDialect extends AbstractDataSourceDialect {
                 if (!bool) {
                     setSubIndexValidFlag(false);
                 }
+            } else if (where instanceof Parenthesis) {
+                Parenthesis whereExpressionParenthesis = (Parenthesis) where;
+                Expression expression = whereExpressionParenthesis.getExpression();
+                setValidityByWhereExpressionsInSchema(expression, masterTableSchema, indexTableSchema);
             } else {
                 setSubIndexValidFlag(false);
                 return;
@@ -831,6 +839,15 @@ public class ClickHouseDialect extends AbstractDataSourceDialect {
 
     }
 
+    private Expression parserException(String originalField) {
+        try {
+            return CCJSqlParserUtil.parseExpression(originalField);
+        } catch (JSQLParserException e) {
+            e.printStackTrace();
+            throw new BusinessException("parseExpression error: " + originalField);
+        }
+    }
+
 
     @Override
     public Dialect executeQuery() {