TSG-2071 管控策略引用subid无法命中v4.3.5-20200601

*限流静态库更新
*Ip归属地日志格式修改
*Maat增加Deffered开关

4 files changed, 64 insertions, 10 deletions

diff --git a/plugin/business/pangu-http/src/pangu_http.cpp b/plugin/business/pangu-http/src/pangu_http.cpp
index 4951ca8..7a9d567 100644
--- a/plugin/business/pangu-http/src/pangu_http.cpp
+++ b/plugin/business/pangu-http/src/pangu_http.cpp
@@ -199,7 +199,7 @@ static Maat_feather_t create_maat_feather(const char * instance_name, const char
 	char redis_port_range[TFE_STRING_MAX] = {0};
 	char accept_tags[TFE_STRING_MAX] = {0};
 	int redis_port_begin=0, redis_port_end=0;
-	int redis_port_select=0;
+	int redis_port_select=0, deferred_load_on=0;
 	int redis_db_idx = 0;
 	char json_cfg_file[TFE_STRING_MAX] = {0}, maat_stat_file[TFE_STRING_MAX] = {0};
 	MESA_load_profile_int_def(profile, section, "maat_input_mode", &(input_mode), 0);
@@ -234,6 +234,7 @@ static Maat_feather_t create_maat_feather(const char * instance_name, const char
 
 	MESA_load_profile_string_def(profile, section, "stat_file", maat_stat_file, sizeof(maat_stat_file), "");
 	MESA_load_profile_int_def(profile, section, "effect_interval_s", &(effect_interval), 60);
+	MESA_load_profile_int_def(profile, section, "deferred_load_on", &(deferred_load_on), 0);
 
 	effect_interval *= 1000;//convert s to ms
 	assert(strlen(inc_cfg_dir) != 0 || strlen(ful_cfg_dir) != 0 || strlen(redis_server)!=0 || strlen(json_cfg_file)!=0);
@@ -269,6 +270,7 @@ static Maat_feather_t create_maat_feather(const char * instance_name, const char
 			Maat_set_feather_opt(target, MAAT_OPT_PERF_ON, NULL, 0);
 		}
 	}
+	Maat_set_feather_opt(target, MAAT_OPT_DEFERRED_LOAD, &deferred_load_on, sizeof(deferred_load_on));
 
 	Maat_set_feather_opt(target, MAAT_OPT_EFFECT_INVERVAL_MS, &effect_interval, sizeof(effect_interval));
 	Maat_set_feather_opt(target, MAAT_OPT_SCAN_DETAIL, &scan_detail, sizeof(scan_detail));
@@ -876,6 +878,43 @@ int maat_table_init(const char* table_name,
 	return table_id;
 }
 
+static char* tfe_unescape(char* s)
+{
+	int i=0,j=0;
+	int len=strlen(s);
+	for(i=0,j=0;i<len;i++)
+	{
+		if(s[i]=='\\')
+		{
+			switch(s[i+1])
+			{
+				case '&':
+					s[j]='&';
+					break;
+				case 'b':
+					s[j]=' ';//space,0x20;
+					break;
+				case '\\':
+					s[j]='\\';
+					break;
+				default:
+					s[j]=s[i];
+					i--;	//undo the followed i++
+					break;
+			}
+			i++;
+			j++;
+		}
+		else
+		{
+			s[j]=s[i];
+			j++;
+		}
+	}
+	s[j]='\0';
+	return s;
+}
+
 void ip_asn_table_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp)
 {
 	int addr_type;
@@ -889,6 +928,7 @@ void ip_asn_table_new_cb(int table_id, const char* key, const char* table_line,
 		TFE_LOG_ERROR(g_pangu_rt->local_logger, "Policy table parse ip ASN failed, ret:%d, %s", ret, table_line);
 		return;
 	}
+	tfe_unescape(organization);
 
 	struct ip_data_table* ip_asn=ALLOC(struct ip_data_table, 1);
 	memset(ip_asn, 0, sizeof(struct ip_data_table));
@@ -923,6 +963,10 @@ void ip_location_table_new_cb(int table_id, const char* key, const char* table_l
 		TFE_LOG_ERROR(g_pangu_rt->local_logger, "Policy table parse ip location failed, ret:%d, %s", ret, table_line);
 		return;
 	}
+	tfe_unescape(continent_full);
+	tfe_unescape(country_full);
+	tfe_unescape(province_full);
+	tfe_unescape(city_full);
 
 	struct ip_data_table* ip_asn=ALLOC(struct ip_data_table, 1);
 	memset(ip_asn, 0, sizeof(struct ip_data_table));
@@ -1118,7 +1162,6 @@ int pangu_policy_init(const char* profile_path, const char* static_section, cons
 			goto error_out;
 		}
 	}
-
 	for(int i = POLICY_ASN_USER_DEFINED; i < POLICY_PROFILE_TABLE_MAX; i++)
 	{
 		ret = maat_ip_table_init(i, ip_table_free_cb, ip_table_dup_cb);
@@ -1127,7 +1170,6 @@ int pangu_policy_init(const char* profile_path, const char* static_section, cons
 			goto error_out;
 		}
 	}
-
 	g_pangu_rt->dyn_maat = create_maat_feather("dyn", profile_path, dynamic_section, g_pangu_rt->thread_num, g_pangu_rt->local_logger);
 	if (!g_pangu_rt->maat)
 	{
@@ -2758,6 +2800,8 @@ int http_ip_location_scan(struct Maat_rule_t *result, struct ip_address *sip, st
 		{
 			hit_cnt+=scan_ret;
 		}
+		memset(buff,0,sizeof(buff));
+		snprintf(buff, sizeof(buff), "%s,%s,%s", ip_location_server->city_full, ip_location_server->province_full, ip_location_server->country_full);
 		ctx->ip_ctx.location_server=tfe_strdup(buff);
 	}
 	if(ip_location_client!=NULL)
@@ -2771,6 +2815,8 @@ int http_ip_location_scan(struct Maat_rule_t *result, struct ip_address *sip, st
 		{
 			hit_cnt+=scan_ret;
 		}
+		memset(buff,0,sizeof(buff));
+		snprintf(buff, sizeof(buff), "%s,%s,%s", ip_location_client->city_full, ip_location_client->province_full, ip_location_client->country_full);
 		ctx->ip_ctx.location_client=tfe_strdup(buff);
 	}
 
@@ -2784,6 +2830,7 @@ int http_ip_location_scan(struct Maat_rule_t *result, struct ip_address *sip, st
 int http_ip_asn_scan(struct Maat_rule_t *result, struct ip_address* sip, struct ip_address* dip, int hit_cnt, unsigned int thread_id, struct pangu_http_ctx * ctx)
 {
 	int scan_ret=0;
+	char buff[TFE_STRING_MAX]={0};
 	struct ip_data_table* ip_asn_client=NULL, *ip_asn_server=NULL;
 
 	Maat_ip_plugin_get_EX_data(g_pangu_rt->maat, g_pangu_rt->plolicy_table_id[POLICY_ASN_USER_DEFINED], sip, (void **)&ip_asn_client, 1);
@@ -2809,7 +2856,8 @@ int http_ip_asn_scan(struct Maat_rule_t *result, struct ip_address* sip, struct
 		{
 			hit_cnt+=scan_ret;
 		}
-		ctx->ip_ctx.asn_server=tfe_strdup(ip_asn_server->asn);
+		snprintf(buff, sizeof(buff), "%s(%s)", ip_asn_server->asn, ip_asn_server->organization);
+		ctx->ip_ctx.asn_server=tfe_strdup(buff);
 	}
 	if(ip_asn_client!=NULL)
 	{
@@ -2821,7 +2869,8 @@ int http_ip_asn_scan(struct Maat_rule_t *result, struct ip_address* sip, struct
 		{
 			hit_cnt+=scan_ret;
 		}
-		ctx->ip_ctx.asn_client=tfe_strdup(ip_asn_client->asn);
+		snprintf(buff, sizeof(buff), "%s(%s)", ip_asn_client->asn, ip_asn_client->organization);
+		ctx->ip_ctx.asn_client=tfe_strdup(buff);
 	}
 	if(ip_asn_server)
 		ip_table_free(ip_asn_server);
@@ -2901,9 +2950,12 @@ void pangu_on_http_begin(const struct tfe_stream * stream,
 	}
 
 	addr_tfe2sapp(stream->addr, &sapp_addr);
-	hit_cnt += Maat_scan_proto_addr(g_pangu_rt->maat, g_pangu_rt->scan_table_id[PXY_CTRL_IP], &sapp_addr, 0,
-		result+hit_cnt, MAX_SCAN_RESULT-hit_cnt, &(ctx->scan_mid), (int) thread_id);
-
+	scan_ret = Maat_scan_proto_addr(g_pangu_rt->maat, g_pangu_rt->scan_table_id[PXY_CTRL_IP], &sapp_addr, 0,
+			result+hit_cnt, MAX_SCAN_RESULT-hit_cnt, &(ctx->scan_mid), (int) thread_id);
+	if(scan_ret>0)
+	{
+		hit_cnt+=scan_ret;
+	}
 	if (hit_cnt > 0)
 	{
 		ctx->action = decide_ctrl_action(result, hit_cnt, &ctx->enforce_rules, &ctx->n_enforce, &ctx->param);
diff --git a/plugin/business/traffic-mirror/src/entry.cpp b/plugin/business/traffic-mirror/src/entry.cpp
index c338698..ff68ea7 100644
--- a/plugin/business/traffic-mirror/src/entry.cpp
+++ b/plugin/business/traffic-mirror/src/entry.cpp
@@ -339,7 +339,7 @@ static Maat_feather_t maat_feather_create_with_override(const char * instance_na
     char redis_port_range[TFE_STRING_MAX] = {0};
     char accept_tags[TFE_STRING_MAX] = {0};
     int redis_port_begin = 0, redis_port_end = 0;
-    int redis_port_select = 0;
+    int redis_port_select = 0, deferred_load_on=0;
     int redis_db_idx = 0;
 
     char json_cfg_file[TFE_STRING_MAX] = {0};
@@ -354,6 +354,7 @@ static Maat_feather_t maat_feather_create_with_override(const char * instance_na
     MESA_load_profile_string_def(profile, section, "maat_redis_server", redis_server, sizeof(redis_server), "");
     MESA_load_profile_string_def(profile, section, "maat_redis_port_range", redis_port_range, sizeof(redis_server),
         "6379");
+	MESA_load_profile_int_def(profile, section, "deferred_load_on", &(deferred_load_on), 0);
 
     ret = sscanf(redis_port_range, "%d-%d", &redis_port_begin, &redis_port_end);
     if (ret == 1)
@@ -414,6 +415,7 @@ static Maat_feather_t maat_feather_create_with_override(const char * instance_na
         }
     }
 
+	Maat_set_feather_opt(target, MAAT_OPT_DEFERRED_LOAD, &deferred_load_on, sizeof(deferred_load_on));
     Maat_set_feather_opt(target, MAAT_OPT_EFFECT_INVERVAL_MS, &effect_interval, sizeof(effect_interval));
     Maat_set_feather_opt(target, MAAT_OPT_SCAN_DETAIL, &scan_detail, sizeof(scan_detail));
     if (strlen(accept_tags) > 0)
diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt
index b897f49..70d764b 100644
--- a/vendor/CMakeLists.txt
+++ b/vendor/CMakeLists.txt
@@ -97,7 +97,7 @@ set_property(TARGET nghttp2-static APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES
 ### ratelimiter
 ExternalProject_Add(ratelimiter PREFIX ratelimiter
         URL ${CMAKE_CURRENT_SOURCE_DIR}/ratelimiter-1.1.0-x86_64.tar.gz
-		URL_MD5 4e5d01586fef4165c4dfd1f1a3595e61
+		URL_MD5 6e4b9b31c70b9e84e2a6434fab6268f9
 		CONFIGURE_COMMAND ""
 		BUILD_COMMAND make
         BUILD_IN_SOURCE 1)
diff --git a/vendor/ratelimiter-1.1.0-x86_64.tar.gz b/vendor/ratelimiter-1.1.0-x86_64.tar.gz
index d127dcc..2a304ad 100644
--- a/vendor/ratelimiter-1.1.0-x86_64.tar.gz
+++ b/vendor/ratelimiter-1.1.0-x86_64.tar.gz