Merge branch 'develop' into push-master

6 files changed, 395 insertions, 184 deletions

diff --git a/bin/tsg_static_tableinfo.conf b/bin/tsg_static_tableinfo.conf
index 44b57dc..549f9a0 100644
--- a/bin/tsg_static_tableinfo.conf
+++ b/bin/tsg_static_tableinfo.conf
@@ -17,25 +17,26 @@
 6	TSG_OBJ_FQDN	expr	UTF8	UTF8	yes	0
 6	TSG_OBJ_FQDN_CAT	expr	UTF8	UTF8	yes	0
 7	TSG_OBJ_KEYWORDS	expr	UTF8	UTF8/GBK	yes	0
-8	TSG_OBJ_HTTP_SIGNATURE	expr_plus	UTF8	UTF8/GBK	yes	0
-9	TSG_FIELD_HTTP_HOST	virtual	TSG_OBJ_FQDN	--
-10	TSG_FIELD_HTTP_URL	virtual	TSG_OBJ_URL	--
-11	TSG_FIELD_HTTP_REQ_HDR	virtual	TSG_OBJ_HTTP_SIGNATURE	--
-12	TSG_FIELD_HTTP_RES_HDR	virtual	TSG_OBJ_HTTP_SIGNATURE	--
-13	TSG_FIELD_HTTP_REQ_CONTENT	virtual	TSG_OBJ_KEYWORDS	--
-14	TSG_FIELD_HTTP_RES_CONTENT	virtual	TSG_OBJ_KEYWORDS	--
-15	TSG_FIELD_SSL_SNI	virtual	TSG_OBJ_FQDN	--
-16	TSG_FIELD_SSL_CN	virtual	TSG_OBJ_FQDN	--
-17	TSG_FIELD_SSL_SAN	virtual	TSG_OBJ_FQDN	--
-18	TSG_FIELD_DNS_QNAME	virtual	TSG_OBJ_FQDN	--
-19	TSG_FIELD_MAIL_ACCOUNT	virtual	TSG_OBJ_ACCOUNT	--
-20	TSG_FIELD_MAIL_FROM	virtual	TSG_OBJ_ACCOUNT	--
-21	TSG_FIELD_MAIL_TO	virtual	TSG_OBJ_ACCOUNT	--
-22	TSG_FIELD_MAIL_SUBJECT	virtual	TSG_OBJ_KEYWORDS	--
-23	TSG_FIELD_MAIL_CONTENT	virtual	TSG_OBJ_KEYWORDS	--
-24	TSG_FIELD_MAIL_ATT_NAME	virtual	TSG_OBJ_KEYWORDS	--
-25	TSG_FIELD_MAIL_ATT_CONTENT	virtual	TSG_OBJ_KEYWORDS	--
-26	TSG_FIELD_FTP_URI	virtual	TSG_OBJ_URL	--
-27	TSG_FIELD_FTP_CONTENT	virtual	TSG_OBJ_KEYWORDS	--
-28	TSG_FIELD_FTP_ACCOUNT	virtual	TSG_OBJ_ACCOUNT	--
-29      FW_PROFILE_DNS_RECORDS  plugin {"key":1,"valid":5}      --
+8	TSG_OBJ_APP_ID	expr	UTF8	UTF8	yes	0
+9	TSG_OBJ_HTTP_SIGNATURE	expr_plus	UTF8	UTF8/GBK	yes	0
+10	TSG_FIELD_HTTP_HOST	virtual	TSG_OBJ_FQDN	--
+11	TSG_FIELD_HTTP_URL	virtual	TSG_OBJ_URL	--
+12	TSG_FIELD_HTTP_REQ_HDR	virtual	TSG_OBJ_HTTP_SIGNATURE	--
+13	TSG_FIELD_HTTP_RES_HDR	virtual	TSG_OBJ_HTTP_SIGNATURE	--
+14	TSG_FIELD_HTTP_REQ_CONTENT	virtual	TSG_OBJ_KEYWORDS	--
+15	TSG_FIELD_HTTP_RES_CONTENT	virtual	TSG_OBJ_KEYWORDS	--
+16	TSG_FIELD_SSL_SNI	virtual	TSG_OBJ_FQDN	--
+17	TSG_FIELD_SSL_CN	virtual	TSG_OBJ_FQDN	--
+18	TSG_FIELD_SSL_SAN	virtual	TSG_OBJ_FQDN	--
+19	TSG_FIELD_DNS_QNAME	virtual	TSG_OBJ_FQDN	--
+20	TSG_FIELD_MAIL_ACCOUNT	virtual	TSG_OBJ_ACCOUNT	--
+21	TSG_FIELD_MAIL_FROM	virtual	TSG_OBJ_ACCOUNT	--
+22	TSG_FIELD_MAIL_TO	virtual	TSG_OBJ_ACCOUNT	--
+23	TSG_FIELD_MAIL_SUBJECT	virtual	TSG_OBJ_KEYWORDS	--
+24	TSG_FIELD_MAIL_CONTENT	virtual	TSG_OBJ_KEYWORDS	--
+25	TSG_FIELD_MAIL_ATT_NAME	virtual	TSG_OBJ_KEYWORDS	--
+26	TSG_FIELD_MAIL_ATT_CONTENT	virtual	TSG_OBJ_KEYWORDS	--
+27	TSG_FIELD_FTP_URI	virtual	TSG_OBJ_URL	--
+28	TSG_FIELD_FTP_CONTENT	virtual	TSG_OBJ_KEYWORDS	--
+29	TSG_FIELD_FTP_ACCOUNT	virtual	TSG_OBJ_ACCOUNT	--
+30      FW_PROFILE_DNS_RECORDS  plugin {"key":1,"valid":5}      --
diff --git a/inc/tsg_rule.h b/inc/tsg_rule.h
index 37a6cbf..5accc0a 100644
--- a/inc/tsg_rule.h
+++ b/inc/tsg_rule.h
@@ -64,15 +64,7 @@ extern Maat_feather_t g_tsg_maat_feather;
 int tsg_rule_init(const char *conffile, void *logger);
 
 int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *a_stream, tsg_protocol_t proto, scan_status_t *mid, Maat_rule_t*result, int result_num);
-
-//return 0 if failed, return >0 on success;
-int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_result_type, Maat_rule_t *result, int result_num, struct _identify_info *identify_info);
-
-//return -1 if failed, return 0 on success;
-int tsg_shared_table_init(const char *conffile, Maat_feather_t maat_feather, void *logger);
-
-//return value: -1: failed, 0: not hit, >0: hit count 
-int tsg_scan_shared_policy(Maat_feather_t maat_feather, void *pkt, int pkt_len, Maat_rule_t *result, int result_num, struct _identify_info *identify_info, scan_status_t *mid, void *logger, int thread_seq);
+int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_result_type, Maat_rule_t*result, int result_num, struct _identify_info *identify_info);
 
 //return NULL if none exists, otherwise return one deny rule;
 struct Maat_rule_t *tsg_fetch_deny_rule(Maat_rule_t *result, int result_num);
diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp
index 418044a..7612d89 100644
--- a/src/tsg_entry.cpp
+++ b/src/tsg_entry.cpp
@@ -3,6 +3,8 @@
 #include <stdlib.h>
 #include <assert.h>
 
+#include <MESA/http.h>
+#include <MESA/ftp.h>
 #include <MESA/stream.h>
 #include <MESA/MESA_prof_load.h>
 #include <MESA/MESA_handle_logger.h>
@@ -12,6 +14,7 @@
 #include "tsg_send_log.h"
 #include "tsg_statistic.h"
 #include "tsg_send_log_internal.h"
+#include "tsg_ssl_utils.h"
 
 #ifdef __cplusplus
 extern "C"
@@ -35,7 +38,7 @@ static __attribute__((__used__)) const char * GIT_VERSION_UNKNOWN = NULL;
 #endif
 
 
-char TSG_MASTER_VERSION_20191226=0;
+char TSG_MASTER_VERSION_20200113=0;
 const char *tsg_conffile="tsgconf/main.conf";
 g_tsg_para_t g_tsg_para;
 
@@ -47,27 +50,74 @@ id2field_t g_tsg_fs2_field[TSG_FS2_MAX]={{TLD_TYPE_UNKNOWN, TSG_FS2_LINKS, "link
 										 {TLD_TYPE_UNKNOWN, TSG_FS2_LOG, "log"},
 										 {TLD_TYPE_UNKNOWN, TSG_FS2_DENY, "deny"}
 	};
+
 static void free_policy_label(int thread_seq, void *project_req_value)
 {
 	dictator_free(thread_seq, project_req_value);
 	project_req_value=NULL;
 }
 
+static char *schema_index2string(tsg_protocol_t proto)
+{
+	char *schema_field_value=NULL;
+
+	switch(proto)
+	{
+		case	PROTO_HTTP:
+			schema_field_value=(char *)"HTTP";
+			break;
+		case	PROTO_SSL:
+			schema_field_value=(char *)"SSL";
+			break;
+		case	PROTO_DNS:
+			schema_field_value=(char *)"DNS";
+			break;
+		case	PROTO_FTP:
+			schema_field_value=(char *)"FTP";
+			break;
+		case	PROTO_BGP:
+			schema_field_value=(char *)"BGP";
+			break;
+		case	PROTO_SIP:
+			schema_field_value=(char *)"SIP";
+			break;
+		case	PROTO_MAIL:
+			schema_field_value=(char *)"MAIL";
+			break;
+		case	PROTO_STREAMING_MEDIA:
+			schema_field_value=(char *)"STREAMING_MEDIA";
+			break;
+		default:
+			break;
+	}
+
+	return schema_field_value;
+}
+
 static int master_send_log(struct streaminfo *a_stream, struct Maat_rule_t *p_result, int result_num, struct _identify_info *identify_info, int thread_seq)
 {
 	tsg_log_t log_msg;	
 	char *domain_field_name=NULL;
 	char *schema_field_name=NULL;
+	char *schema_field_value=NULL;
 	struct TLD_handle_t *TLD_handle=NULL;
 
 	TLD_handle=TLD_create(thread_seq);
-	if(identify_info!=NULL)
+	if(identify_info!=NULL && (identify_info->proto>PROTO_UNKONWN) && (identify_info->proto<PROTO_MAX))
 	{
 		schema_field_name=log_field_id2name(g_tsg_log_instance, LOG_COMMON_SCHAME_TYPE);
-		TLD_append(TLD_handle, schema_field_name, (void *)((identify_info->proto==PROTO_HTTP) ? "HTTP" : "SSL"), TLD_TYPE_STRING);
-		
-		domain_field_name=log_field_id2name(g_tsg_log_instance, ((identify_info->proto==PROTO_HTTP) ? LOG_HTTP_HOST : LOG_SSL_SNI));
-		TLD_append(TLD_handle, domain_field_name, (void *)identify_info->domain, TLD_TYPE_STRING);
+
+		schema_field_value=schema_index2string(identify_info->proto);
+		if(schema_field_value!=NULL)
+		{
+			TLD_append(TLD_handle, schema_field_name, (void *)schema_field_value, TLD_TYPE_STRING);
+		}
+
+		if(identify_info->proto==PROTO_HTTP || identify_info->proto==PROTO_SSL)
+		{
+			domain_field_name=log_field_id2name(g_tsg_log_instance, ((identify_info->proto==PROTO_HTTP) ? LOG_HTTP_HOST : LOG_SSL_SNI));
+			TLD_append(TLD_handle, domain_field_name, (void *)identify_info->domain, TLD_TYPE_STRING);
+		}
 	}
 	
 	log_msg.a_stream=a_stream;
@@ -110,9 +160,94 @@ static struct Maat_rule_t *tsg_policy_decision_criteria(Maat_rule_t *result, int
 	return p_result;
 }
 
+static int identify_application_protocol(struct streaminfo *a_stream, struct _identify_info *identify_info)
+{
+	int ret=0;
+
+	identify_info->proto = PROTO_UNKONWN;
+	//http
+	char *host = NULL;
+	ret=http_host_parser((char *)a_stream->ptcpdetail->pdata, (unsigned int)a_stream->ptcpdetail->datalen, DIR_C2S, &host);
+	if(ret>=0)
+	{
+		identify_info->proto=PROTO_HTTP;
+		if(ret==0)
+		{
+			identify_info->domain_len=0;
+		}
+		else
+		{
+			identify_info->domain_len=MIN(ret, (int)sizeof(identify_info->domain) - 1);
+			strncpy(identify_info->domain, host, identify_info->domain_len);
+		}
+		return 1;
+	}
+	
+	//ssl
+	enum chello_parse_result chello_status = CHELLO_PARSE_INVALID_FORMAT;
+	struct ssl_chello *chello = NULL;
+
+	chello=ssl_chello_parse((unsigned char *)a_stream->ptcpdetail->pdata, (unsigned int)a_stream->ptcpdetail->datalen, &chello_status);
+	if(chello_status==CHELLO_PARSE_SUCCESS)
+	{
+		identify_info->proto=PROTO_SSL;
+		if(chello->sni==NULL)
+		{
+			identify_info->domain_len = 0;
+		}
+		else
+		{
+			identify_info->domain_len = strnlen(chello->sni, sizeof(identify_info->domain) - 1);
+			strncpy(identify_info->domain, chello->sni, identify_info->domain_len);
+		}
+		
+		ssl_chello_free(chello);
+		return 1;
+	}
+	
+    ssl_chello_free(chello);
+
+	//dns
+	struct stream_tuple4_v4 *tpl4 = NULL;
+	struct stream_tuple4_v6 *tpl6 = NULL;
+
+	switch(a_stream->addr.addrtype)
+	{
+		case ADDR_TYPE_IPV4:
+			tpl4=a_stream->addr.tuple4_v4;
+			if((ntohs(tpl4->source)==53) || (ntohs(tpl4->dest)==53))
+			{
+				identify_info->proto=PROTO_DNS;
+				return 1;
+			}
+			break;
+		case ADDR_TYPE_IPV6:
+			tpl6=a_stream->addr.tuple4_v6;
+			if((ntohs(tpl6->source)==53) || (ntohs(tpl6->dest)==53))
+			{
+				identify_info->proto=PROTO_DNS;
+				return 1;
+			}
+			break;
+		default:
+			break;
+	}
+
+	//ftp
+	ret=ftp_control_identify(a_stream);
+	if(ret>0)
+	{
+		identify_info->proto=PROTO_FTP;
+		return 1;
+	}
+
+	//mail
+	
+	return ret;
+}
+
 extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int thread_seq,void *a_packet)
 {
-	int identify_flag=0;
 	int ret=0,hit_num=0;
 	int state=APP_STATE_DROPME;
 	scan_status_t mid=NULL;
@@ -128,40 +263,19 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
 		case	OP_STATE_PENDING:
 			FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_LINKS], 0, FS_OP_ADD, 1);
 			
-			ret=tsg_scan_nesting_addr(g_tsg_maat_feather, a_tcp, PROTO_MAX, &mid, all_result+hit_num, MAX_RESULT_NUM-hit_num);
+			memset(&identify_info, 0, sizeof(identify_info));
+			identify_application_protocol(a_tcp, &identify_info);
+			
+			ret=tsg_scan_nesting_addr(g_tsg_maat_feather, a_tcp, identify_info.proto, &mid, all_result+hit_num, MAX_RESULT_NUM-hit_num);
 			if(ret>0)
 			{
 				hit_num+=ret;
 				q_result=tsg_policy_decision_criteria(all_result, hit_num);
-				FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_ADDR], 0, FS_OP_ADD, 1);				
-				MESA_handle_runtime_log(g_tsg_para.logger,
-										RLOG_LV_DEBUG,
-										"SCAN_IP",
-										"Hit addr: %s scan ret: %d policy_id: %d service: %d action: %d",
-										printaddr(&a_tcp->addr, thread_seq),
-										ret,
-										q_result->config_id,
-										q_result->service_id,
-										q_result->action);			
+				FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_ADDR], 0, FS_OP_ADD, 1);		
 			}
-			else
-			{
-				MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "SCAN_IP", "Not hit %s scan ret: %d",
-printaddr(&a_tcp->addr, thread_seq), ret);			
-			}
-
-
-			memset(&identify_info, 0, sizeof(identify_info));
 			
-			ret=tsg_scan_shared_policy(g_tsg_maat_feather, 
-										a_tcp->ptcpdetail->pdata,
-										a_tcp->ptcpdetail->datalen,
-										all_result+hit_num,
-										MAX_RESULT_NUM-hit_num,
-										&identify_info,
-										&mid,
-										g_tsg_para.logger,
-										thread_seq);
+			ret=tsg_scan_shared_policy(g_tsg_maat_feather, &identify_info, all_result+hit_num, MAX_RESULT_NUM-hit_num, &mid, thread_seq);
+
 			if(ret>0)
 			{
 				FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_SHARE], 0, FS_OP_ADD, 1);
@@ -179,7 +293,6 @@ printaddr(&a_tcp->addr, thread_seq), ret);
 										);
 
 				hit_num+=ret;
-				identify_flag=1;
 			}
 			else
 			{
@@ -204,11 +317,18 @@ printaddr(&a_tcp->addr, thread_seq), ret);
 					case	TSG_ACTION_DENY:
 						MESA_kill_tcp(a_tcp, a_packet);
 						FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_DENY], 0, FS_OP_ADD, 1);
-						MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "DENY", "Hit deny policy, policy_id: %d action: %d addr: %s",
-															p_result[0].config_id, p_result[0].action, printaddr(&a_tcp->addr, thread_seq));
 
-						master_send_log(a_tcp, p_result, 1, ((identify_flag==1) ? &identify_info : NULL), thread_seq);
+						master_send_log(a_tcp, p_result, 1, &identify_info, thread_seq);
 						state|=APP_STATE_DROPPKT|APP_STATE_KILL_OTHER;
+
+						MESA_handle_runtime_log(g_tsg_para.logger,
+												RLOG_LV_DEBUG,
+												"DENY",
+												"Hit deny policy, policy_id: %d action: %d addr: %s",
+												p_result[0].config_id,
+												p_result[0].action,
+												printaddr(&a_tcp->addr, thread_seq)
+												);
 						break;
 					case	TSG_ACTION_MONITOR:
 						if(q_result!=NULL && (p_result==q_result))
@@ -263,6 +383,12 @@ printaddr(&a_tcp->addr, thread_seq), ret);
 						break;
 				}
 			}
+
+			if(mid!=NULL)
+			{
+				Maat_clean_status(&mid);
+				mid=NULL;
+			}
 		break;
 		case	OP_STATE_DATA:
 		case	OP_STATE_CLOSE:
diff --git a/src/tsg_entry.h b/src/tsg_entry.h
index 628076c..90d77df 100644
--- a/src/tsg_entry.h
+++ b/src/tsg_entry.h
@@ -5,6 +5,21 @@
 #include <MESA/field_stat2.h>
 #include "tsg_rule.h"
 
+
+#ifndef MIN
+#define MIN(a, b)  	(((a) < (b)) ? (a) : (b))
+#endif
+
+enum MASTER_TABLE{
+	TABLE_IP_ADDR=0,
+	TABLE_SUBSCRIBER_ID,
+	TABLE_APP_ID,
+	TABLE_HTTP_HOST,
+	TABLE_SSL_SNI,
+	TABLE_MAX
+};
+
+
 enum TSG_FS2_TYPE{
 	TSG_FS2_LINKS=0,
 	TSG_FS2_BYPASS,
@@ -40,14 +55,15 @@ struct _master_context
 	struct Maat_rule_t *result;
 };
 
+#define	_MAX_TABLE_NAME_LEN	64
 typedef	struct _tsg_para
 {
-	int device_id;
-	int ip_addr_table_id;
-	int subscribe_id_table_id;
+	int device_id;	
+	int table_id[TABLE_MAX];
 	int dyn_subscribe_ip_table_id; //TSG_DYN_SUBSCRIBER_IP
 	int priority_project_id;
 	int fs2_field_id[TSG_FS2_MAX];
+	char table_name[TABLE_MAX][_MAX_TABLE_NAME_LEN];
 	void *logger;
 	screen_stat_handle_t fs2_handle;
 }g_tsg_para_t;
@@ -125,4 +141,6 @@ typedef	struct _tsg_statistic
 
 int tsg_statistic_init(const char *conffile, void *logger);
 
+int tsg_scan_shared_policy(Maat_feather_t maat_feather, struct _identify_info *identify_info, Maat_rule_t *result, int result_num, scan_status_t *mid, int thread_seq);
+
 #endif
diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp
index a7ad96c..2e62ff2 100644
--- a/src/tsg_rule.cpp
+++ b/src/tsg_rule.cpp
@@ -11,7 +11,6 @@
 #include "Maat_rule.h"
 #include "Maat_command.h"
 #include "MESA/http.h"
-#include "tsg_ssl_utils.h"
 #include "tsg_rule.h"
 #include "tsg_entry.h"
 
@@ -21,10 +20,6 @@ Maat_feather_t g_tsg_dynamic_maat_feather;
 #define	MAX_PATH_LEN	1024
 #define	MAX_IPV6_ADDR_LEN	128
 
-#ifndef MIN
-#define MIN(a, b)  	(((a) < (b)) ? (a) : (b))
-#endif
-
 enum kni_scan_table{
     TSG_FIELD_SSL_SNI,
 	TSG_FIELD_HTTP_HOST,
@@ -42,6 +37,21 @@ const struct _str2index method2index[TSG_METHOD_TYPE_MAX]={	{TSG_METHOD_TYPE_UNK
 													{TSG_METHOD_TYPE_RESET, 3, (char *)"rst"}
 	};
 
+const struct _str2index g_tsg_proto_string[PROTO_MAX+1]={{PROTO_UNKONWN, 0, (char *)""}, 
+										 {PROTO_IPv4, 5, (char *)"IPv4."},
+										 {PROTO_IPv6, 5, (char *)"IPv6."},
+										 {PROTO_TCP, 4, (char *)"TCP."},
+										 {PROTO_UDP, 4, (char *)"UDP."},
+										 {PROTO_HTTP, 5, (char *)"HTTP."},
+										 {PROTO_MAIL, 5, (char *)"MAIL."},
+										 {PROTO_DNS, 4, (char *)"DNS."},
+										 {PROTO_FTP, 4, (char *)"FTP."},
+										 {PROTO_SSL, 4, (char *)"SSL."},
+										 {PROTO_SIP, 4, (char *)"SIP."},
+										 {PROTO_BGP, 4, (char *)"BGP."},
+										 {PROTO_STREAMING_MEDIA, 16, (char *)"STREAMING_MEDIA."},
+										 {PROTO_MAX, 0, (char *)""}
+};
 
 void subscribe_id_dup_data(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void* argp)
 {
@@ -50,7 +60,7 @@ void subscribe_id_dup_data(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX
 	*to=calloc(1, strlen((char *)*from)+1);
 	memcpy(*to, *from, strlen((char *)*from));
 	
-	MESA_handle_runtime_log(logger, RLOG_LV_INFO, "SUBSCRIBE_ID", "Dup subscribe_id: %s table_id: %d", (char *)*to, table_id);
+	MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "SUBSCRIBE_ID", "Dup subscribe_id: %s table_id: %d", (char *)*to, table_id);
 	return;
 }
 
@@ -80,7 +90,7 @@ void subscribe_id_new_data(int table_id, const char* key, const char* table_line
 	
 	memcpy(*ad, subscribe_id, strlen(subscribe_id));
 	MESA_handle_runtime_log(logger, 
-								RLOG_LV_INFO,
+								RLOG_LV_DEBUG,
 								"SUBSCRIBE_ID",
 								"Add subscribe_id: %s table_id: %d key: %s table_line: %s",
 								*ad,
@@ -94,7 +104,7 @@ void subscribe_id_new_data(int table_id, const char* key, const char* table_line
 void subscribe_id_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp)
 {
 	void *logger=argp;
-	MESA_handle_runtime_log(logger, RLOG_LV_INFO, "SUBSCRIBE_ID", "Delete subscribe_id: %s table_id: %d", (char *)*ad, table_id);
+	MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "SUBSCRIBE_ID", "Delete subscribe_id: %s table_id: %d", (char *)*ad, table_id);
 	free(*ad);
 	*ad=NULL;
 
@@ -200,16 +210,16 @@ static Maat_feather_t init_maat_feather(const char* conffile, char* instance_nam
 
 int tsg_rule_init(const char* conffile, void *logger)
 {
-	int ret=0;
+	int i=0,ret=0;
 	char maat_conffile[256]={0};
-	char ip_addr_table[32]={0};
-	char subscriber_id_table[32]={0};
 	char cb_subscriber_ip_table[32]={0};
 	
 	MESA_load_profile_string_def(conffile, "MAAT", "PROFILE", maat_conffile, sizeof(maat_conffile), "./tsgconf/maat_profile.conf");	
-	MESA_load_profile_string_def(conffile, "MAAT", "IP_ADDR_TABLE", ip_addr_table, sizeof(ip_addr_table), "TSG_OBJ_IP_ADDR");
-	MESA_load_profile_string_def(conffile, "MAAT", "SUBSCRIBER_ID_TABLE", subscriber_id_table, sizeof(subscriber_id_table), "TSG_OBJ_SUBSCRIBER_ID");
-
+	MESA_load_profile_string_def(conffile, "MAAT", "IP_ADDR_TABLE", g_tsg_para.table_name[TABLE_IP_ADDR], _MAX_TABLE_NAME_LEN, "TSG_OBJ_IP_ADDR");
+	MESA_load_profile_string_def(conffile, "MAAT", "SUBSCRIBER_ID_TABLE", g_tsg_para.table_name[TABLE_SUBSCRIBER_ID], _MAX_TABLE_NAME_LEN, "TSG_OBJ_SUBSCRIBER_ID");
+	MESA_load_profile_string_def(conffile, "MAAT", "APP_ID_TABLE", g_tsg_para.table_name[TABLE_APP_ID], _MAX_TABLE_NAME_LEN, "TSG_OBJ_APP_ID");
+	MESA_load_profile_string_def(conffile, "MAAT", "HTTP_HOST_TABLE", g_tsg_para.table_name[TABLE_HTTP_HOST], _MAX_TABLE_NAME_LEN, "TSG_FIELD_HTTP_HOST");
+	MESA_load_profile_string_def(conffile, "MAAT", "SSL_SNI_TABLE", g_tsg_para.table_name[TABLE_SSL_SNI], _MAX_TABLE_NAME_LEN, "TSG_FIELD_SSL_SNI");
 
 	//init dynamic maat feather
 	g_tsg_maat_feather=init_maat_feather(maat_conffile, (char *)"TSG_STATIC", (char *)"STATIC", logger);
@@ -218,29 +228,16 @@ int tsg_rule_init(const char* conffile, void *logger)
 		MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "init_maat_feather failed, instance_name: %s module: %s", "TSG_STATIC", "STATIC"); 
 		return -1;
 	}
-	
-	g_tsg_para.ip_addr_table_id=Maat_table_register(g_tsg_maat_feather, ip_addr_table);
-	if(g_tsg_para.ip_addr_table_id<0)
-	{
-		MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "Maat_table_register %s failed", ip_addr_table); 
-		return -1;
-	}
-	
-	g_tsg_para.subscribe_id_table_id=Maat_table_register(g_tsg_maat_feather, subscriber_id_table);
-	if(g_tsg_para.subscribe_id_table_id<0)
-	{
-		MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "RULE_INIT", "Maat_table_register %s failed", subscriber_id_table); 
-		return -1;
-	}
 
-	// init sni or host share table
-	ret=tsg_shared_table_init(conffile, g_tsg_maat_feather, logger);
-	if(ret<0)
+	for(i=0; i<TABLE_MAX; i++)
 	{
-		MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "RULE_INIT", "tsg_shared_table_init %s failed"); 
-		return -1;	
-	}
-	
+		g_tsg_para.table_id[i]=Maat_table_register(g_tsg_maat_feather, g_tsg_para.table_name[i]);
+		if(g_tsg_para.table_id[i]<0)
+		{
+			MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "Maat_table_register %s failed, Please check tsgconf/tsg_static_tableinfo.conf", g_tsg_para.table_name[i]); 
+			return -1;
+		}
+	}	
 
 	//init dynamic maat feather
 	g_tsg_dynamic_maat_feather=init_maat_feather(maat_conffile, (char *)"TSG_DYNAMIC", (char *)"DYNAMIC", logger);
@@ -275,59 +272,6 @@ int tsg_rule_init(const char* conffile, void *logger)
 	return 0;
 }
 
-static void protocol_identify(char *buff, int buff_len, struct _identify_info *result){
-    result->proto = PROTO_UNKONWN;
-	//http
-	char *host = NULL;
-	int ret = http_host_parser(buff, (uint32_t)buff_len, DIR_C2S, &host);
-	//printf("http_host_parse: ret = %d, buff_len = %d, buff = %s\n", ret, buff_len, buff);
-	if(ret >= 0){
-		result->proto = PROTO_HTTP;
-		if(ret == 0){
-			result->domain_len = 0;
-		}
-		else{
-			result->domain_len = MIN(ret, (int)sizeof(result->domain) - 1);
-			strncpy(result->domain, host, result->domain_len);
-		}
-		return;
-	}
-	//ssl
-	enum chello_parse_result chello_status = CHELLO_PARSE_INVALID_FORMAT;
-	struct ssl_chello *chello = NULL;
-	chello = ssl_chello_parse((const unsigned char*)buff, buff_len, &chello_status);
-	if(chello_status == CHELLO_PARSE_SUCCESS){
-		result->proto = PROTO_SSL;
-		if(chello->sni == NULL){
-			result->domain_len = 0;
-		}
-		else{
-			result->domain_len = strnlen(chello->sni, sizeof(result->domain) - 1);
-			strncpy(result->domain, chello->sni, result->domain_len);
-		}
-	}
-    ssl_chello_free(chello);
-	return;
-}
-
-//return -1 if failed, return 0 on success;
-int tsg_shared_table_init(const char *conffile, Maat_feather_t maat_feather, void *logger){
-    g_tsg_maat_feather = maat_feather;
-    g_kni_scan_table_name[TSG_FIELD_HTTP_HOST] = "TSG_FIELD_HTTP_HOST";
-    g_kni_scan_table_name[TSG_FIELD_SSL_SNI] = "TSG_FIELD_SSL_SNI";
-    int i;
-    for(i = 0; i < SCAN_TABLE_MAX; i++){
-		g_kni_scan_tableid[i] = Maat_table_register(maat_feather, g_kni_scan_table_name[i]);
-		if(g_kni_scan_tableid[i] < 0){
-            MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "Failed at Maat_table_register, tablename = %s, ret = %d", 
-				g_kni_scan_table_name[i], g_kni_scan_tableid[i]);
-			return -1;
-		}
-	}
-    return 0;
-}
-
-
 int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_result_type, Maat_rule_t*result, int result_num, struct _identify_info *identify_info)
 {
 	int num=0;
@@ -420,8 +364,13 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
 	const struct streaminfo *cur_stream = a_stream;
 
 
-	if(result == NULL || result_num <= 0 || a_stream == NULL || maat_feather == NULL)
+	if(result==NULL || result_num<=0 || a_stream==NULL || maat_feather==NULL)
 	{
+		MESA_handle_runtime_log(g_tsg_para.logger,
+								RLOG_LV_FATAL,
+								"SCAN_NESTING_ADDR",
+								"result==NULL || result_num<=0 || maat_feather==NULL || a_stream==%s",
+								(a_stream!=NULL) ? printaddr(&a_stream->addr, a_stream->threadnum) : "NULL");
 		return -1;
 	}
 	
@@ -466,17 +415,38 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
 			}
 			
  			maat_ret=Maat_scan_proto_addr(maat_feather,
- 											g_tsg_para.ip_addr_table_id,
+ 											g_tsg_para.table_id[TABLE_IP_ADDR],
  											p_addr,
  											tans_proto,
  											result+hit_num,
  											result_num-hit_num,
  											mid,
  											cur_stream->threadnum);
-			if(maat_ret > 0)
+			if(maat_ret>0)
 			{
+				MESA_handle_runtime_log(g_tsg_para.logger,
+											RLOG_LV_DEBUG,
+											"SCAN_IP",
+											"Hit addr: %s scan ret: %d policy_id: %d service: %d action: %d",
+											printaddr(&cur_stream->addr, cur_stream->threadnum),
+											maat_ret,
+											result[hit_num].config_id,
+											result[hit_num].service_id,
+											result[hit_num].action
+										);
+
 				hit_num+=maat_ret;
 			}
+			else
+			{
+				MESA_handle_runtime_log(g_tsg_para.logger,
+											RLOG_LV_DEBUG,
+											"SCAN_IP",
+											"No hit addr: %s scan ret: %d",
+											printaddr(&cur_stream->addr, cur_stream->threadnum),
+											maat_ret
+										);	
+			}
 		}
 		
 		cur_stream = cur_stream->pfather;
@@ -484,6 +454,48 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
 	}while(cur_stream != NULL && hit_num < result_num);
 
 
+	if(hit_num<result_num && proto>PROTO_UNKONWN && proto<PROTO_MAX)
+	{
+		maat_ret=Maat_full_scan_string(maat_feather,
+									g_tsg_para.table_id[TABLE_APP_ID],
+									CHARSET_GBK,
+									g_tsg_proto_string[proto].type,
+									strlen(g_tsg_proto_string[proto].type),
+									result+hit_num,
+									&found_pos,
+									result_num-hit_num,
+									mid,
+									a_stream->threadnum);
+		if(maat_ret > 0)
+		{
+			MESA_handle_runtime_log(g_tsg_para.logger,
+									RLOG_LV_DEBUG,
+									"SCAN_PROTO",
+									"Hit PROTO: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
+									g_tsg_proto_string[proto].type,
+									maat_ret,
+									result[hit_num].config_id,
+									result[hit_num].service_id,
+									result[hit_num].action,
+									printaddr(&a_stream->addr, a_stream->threadnum)
+									);
+
+			hit_num+=maat_ret;
+		}
+		else
+		{
+			MESA_handle_runtime_log(g_tsg_para.logger,
+									RLOG_LV_DEBUG,
+									"SCAN_PROTO",
+									"No hit PROTO: %s scan ret: %d addr: %s",
+									g_tsg_proto_string[proto].type,
+									maat_ret,
+									printaddr(&a_stream->addr, a_stream->threadnum)
+									);
+		}
+	}
+
+
 	if(hit_num<result_num)
 	{
 		tsg_get_subscribe_id(a_stream, &source_subscribe_id, &dest_subscribe_id);
@@ -491,7 +503,7 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
 		if(source_subscribe_id!=NULL)
 		{
 			maat_ret=Maat_full_scan_string(maat_feather,
-										g_tsg_para.subscribe_id_table_id,
+										g_tsg_para.table_id[TABLE_SUBSCRIBER_ID],
 										CHARSET_GBK,
 										source_subscribe_id,
 										strlen(source_subscribe_id),
@@ -501,9 +513,32 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
 										mid,
 										a_stream->threadnum);
 			if(maat_ret > 0)
-			{
+			{				
+				MESA_handle_runtime_log(g_tsg_para.logger,
+											RLOG_LV_DEBUG,
+											"SCAN_SUBSCRIBER",
+											"Hit source subscribe id: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
+											source_subscribe_id,
+											maat_ret,
+											result[hit_num].config_id,
+											result[hit_num].service_id,
+											result[hit_num].action,
+											printaddr(&a_stream->addr, a_stream->threadnum)
+										);
+
 				hit_num+=maat_ret;
 			}
+			else
+			{
+				MESA_handle_runtime_log(g_tsg_para.logger,
+											RLOG_LV_DEBUG,
+											"SCAN_SUBSCRIBER",
+											"No hit source subscribe id: %s scan ret: %d addr: %s",
+											source_subscribe_id,
+											maat_ret,
+											printaddr(&a_stream->addr, a_stream->threadnum)
+										);
+			}
 			
 			subscribe_id_free_data(g_tsg_para.dyn_subscribe_ip_table_id,(MAAT_PLUGIN_EX_DATA *)&source_subscribe_id, 0, g_tsg_para.logger);
 		}
@@ -511,7 +546,7 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
 		if(dest_subscribe_id!=NULL)
 		{
 			maat_ret=Maat_full_scan_string(maat_feather,
-										g_tsg_para.subscribe_id_table_id,
+										g_tsg_para.table_id[TABLE_SUBSCRIBER_ID],
 										CHARSET_GBK,
 										dest_subscribe_id,
 										strlen(dest_subscribe_id),
@@ -522,8 +557,31 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
 										a_stream->threadnum);
 			if(maat_ret > 0)
 			{
+				MESA_handle_runtime_log(g_tsg_para.logger,
+											RLOG_LV_DEBUG,
+											"SCAN_SUBSCRIBER",
+											"Hit dest subscribe id: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
+											dest_subscribe_id,
+											maat_ret,
+											result[hit_num].config_id,
+											result[hit_num].service_id,
+											result[hit_num].action,
+											printaddr(&a_stream->addr, a_stream->threadnum)
+										);
+
 				hit_num+=maat_ret;
 			}
+			else
+			{
+				MESA_handle_runtime_log(g_tsg_para.logger,
+											RLOG_LV_DEBUG,
+											"SCAN_SUBSCRIBER",
+											"No hit dest subscribe id: %s scan ret: %d addr: %s",
+											dest_subscribe_id,
+											maat_ret,
+											printaddr(&a_stream->addr, a_stream->threadnum)
+										);
+			}
 			
 			subscribe_id_free_data(g_tsg_para.dyn_subscribe_ip_table_id,(MAAT_PLUGIN_EX_DATA *)&dest_subscribe_id, 0, g_tsg_para.logger);
 		}
@@ -534,23 +592,39 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
 
 
 //return value: -1: failed, 0: not hit, >0: hit count 
-int tsg_scan_shared_policy(Maat_feather_t maat_feather, void *pkt, int pkt_len, Maat_rule_t *result, int result_num, 
-			struct _identify_info *identify_info, scan_status_t *mid, void *logger, int thread_seq)
+int tsg_scan_shared_policy(Maat_feather_t maat_feather, struct _identify_info *identify_info, Maat_rule_t *result, int result_num,  scan_status_t *mid, int thread_seq)
 {
-    memset(identify_info, 0, sizeof(*identify_info));
-    protocol_identify((char*)pkt, pkt_len, identify_info);
-	if(identify_info->proto != PROTO_SSL && identify_info->proto != PROTO_HTTP){
-		return -1;
-	}
-	int tableid;
-	if(identify_info->proto == PROTO_SSL){
-		tableid = g_kni_scan_tableid[TSG_FIELD_SSL_SNI];
-	}
-	else{
-		tableid = g_kni_scan_tableid[TSG_FIELD_HTTP_HOST];
+	int ret=0,idx=0;
+	
+	if(identify_info->proto!=PROTO_UNKONWN && identify_info->domain_len>0)
+	{
+		switch(identify_info->proto)
+		{
+			case	PROTO_HTTP:
+				idx=TABLE_HTTP_HOST;
+				break;
+			case	PROTO_SSL:
+				idx=TABLE_SSL_SNI;
+				break;
+			default:
+				return 0;
+				break;
+		}
+		
+		ret=Maat_full_scan_string(g_tsg_maat_feather,
+									g_tsg_para.table_id[idx],
+									CHARSET_UTF8,
+									identify_info->domain,
+									identify_info->domain_len,
+									result,
+									NULL,
+									result_num,
+									mid,
+									thread_seq
+								);
 	}
-	return Maat_full_scan_string(g_tsg_maat_feather, tableid, CHARSET_UTF8, identify_info->domain, identify_info->domain_len,
-									result, NULL, result_num, mid, thread_seq);
+
+	return ret;
 }
 
 
diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp
index 246cdb8..e21a822 100644
--- a/src/tsg_send_log.cpp
+++ b/src/tsg_send_log.cpp
@@ -360,8 +360,8 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
 	}
 
 	MESA_load_profile_int_def(conffile, "TSG_LOG", "MAX_SERVICE",&(_instance->max_service), 0);
-	(_instance->topic_rkt)=(rd_kafka_topic_t **)calloc(1, sizeof(rd_kafka_topic_t*));
-	*(_instance->topic_rkt)=(rd_kafka_topic_t *)calloc(1, (1+_instance->max_service)*sizeof(rd_kafka_topic_t*));
+	//(_instance->topic_rkt)=(rd_kafka_topic_t **)calloc(1, sizeof(void *));
+	(_instance->topic_rkt)=(rd_kafka_topic_t **)calloc(1, (1+_instance->max_service)*sizeof(rd_kafka_topic_t*));
 	
 	_instance->service2topic=(id2field_t *)calloc(1, (1+_instance->max_service)*sizeof(id2field_t));