TSG-17862 Proxy支持Internal IP address和Exteral IP Address的扫描, 支持以Topic方式上传HTTP请求体/应答体v4.8.51-20231215

3 files changed, 38 insertions, 78 deletions

diff --git a/plugin/business/doh/src/doh.cpp b/plugin/business/doh/src/doh.cpp
index f8b4ba9..b6c7ce2 100644
--- a/plugin/business/doh/src/doh.cpp
+++ b/plugin/business/doh/src/doh.cpp
@@ -27,15 +27,6 @@ struct doh_action_param
 	pthread_mutex_t lock;
 };
 
-struct doh_app_id_dict
-{
-	int ref_cnt;
-	int app_id;
-	long long int group_id;
-
-	pthread_mutex_t lock;
-};
-
 struct dns_str2idx
 {
 	int index;
@@ -151,29 +142,6 @@ void doh_action_param_free_cb(int table_id, void **ad, long argl, void *argp)
 	return;
 }
 
-void doh_app_dict_table_free_cb(int table_id, void **ad, long argl, void* argp)
-{
-	if(*ad==NULL)
-	{
-		return;
-	}
-
-	struct doh_app_id_dict *app_dict=(struct doh_app_id_dict *)(*ad);
-	pthread_mutex_lock(&(app_dict->lock));
-	app_dict->ref_cnt--;
-	if(app_dict->ref_cnt>0)
-	{
-		pthread_mutex_unlock(&(app_dict->lock));
-		return;
-	}
-	pthread_mutex_unlock(&(app_dict->lock));
-	pthread_mutex_destroy(&(app_dict->lock));
-
-	FREE(&app_dict);
-	*ad=NULL;
-	return;
-}
-
 static void doh_get_cheat_data(long long p_result, int qtype, struct doh_ctx *ctx, const char *str_stream_info)
 {
 	int i;
@@ -339,6 +307,12 @@ static void doh_maat_scan(const struct tfe_stream *stream, const struct tfe_http
             TFE_LOG_INFO(g_doh_conf->local_logger, "Scan %s, NO hit host: %s scan ret: %d addr: %s",
                          g_doh_conf->tables[TYPE_HOST].name, host, scan_ret, stream->str_stream_info);
         }
+		scan_ret = maat_scan_not_logic(g_doh_conf->maat, g_doh_conf->tables[TYPE_HOST].id,
+									   result + hit_cnt, MAX_SCAN_RESULT - hit_cnt, &n_hit_result, ctx->scan_mid);
+        if (scan_ret == MAAT_SCAN_HIT)
+        {
+			hit_cnt += n_hit_result;
+        }
 
         scan_ret = tfe_scan_fqdn_cat(stream, result, ctx->scan_mid, hit_cnt, g_doh_conf->local_logger, g_doh_conf->tables[TYPE_HOST_CAT].id);
         if( scan_ret > 0)
@@ -351,61 +325,35 @@ static void doh_maat_scan(const struct tfe_stream *stream, const struct tfe_http
 	doh_addr_tfe2sapp(stream->addr, &sapp_addr);
 	if (sapp_addr.addrtype == ADDR_TYPE_IPV4)
 	{
-		scan_ret = maat_scan_ipv4(g_doh_conf->maat, g_doh_conf->tables[TYPE_SRC_ADDR].id,sapp_addr.v4->saddr,
-								  sapp_addr.v4->source, 6, result+hit_cnt, MAX_SCAN_RESULT-hit_cnt,
-								  &n_hit_result, ctx->scan_mid);
-		if (n_hit_result == MAAT_SCAN_HIT)
+		scan_ret = tfe_scan_ipv4_addr(result, ctx->scan_mid, hit_cnt, sapp_addr);
+		if (scan_ret > 0)
 		{
-			hit_cnt += n_hit_result;
+			hit_cnt += scan_ret;
 		}
-		scan_ret = maat_scan_ipv4(g_doh_conf->maat, g_doh_conf->tables[TYPE_DST_ADDR].id,sapp_addr.v4->daddr,
-								  sapp_addr.v4->dest, 6, result+hit_cnt, MAX_SCAN_RESULT-hit_cnt,
-								  &n_hit_result, ctx->scan_mid);
-
-		if(scan_ret == MAAT_SCAN_HIT)
+		scan_ret = tfe_scan_ipv4_internal_addr(stream, result, ctx->scan_mid, hit_cnt, sapp_addr);
+		if (scan_ret > 0)
 		{
-			hit_cnt += n_hit_result;
+			hit_cnt += scan_ret;
 		}
 	}
 	if (sapp_addr.addrtype == ADDR_TYPE_IPV6)
 	{
-		scan_ret = maat_scan_ipv6(g_doh_conf->maat, g_doh_conf->tables[TYPE_SRC_ADDR].id, sapp_addr.v6->saddr,
-								 sapp_addr.v6->source, 6, result+hit_cnt, MAX_SCAN_RESULT-hit_cnt,
-								 &n_hit_result, ctx->scan_mid);
-		if (scan_ret == MAAT_SCAN_HIT)
+		scan_ret = tfe_scan_ipv6_addr(result, ctx->scan_mid, hit_cnt, sapp_addr);
+		if (scan_ret > 0)
 		{
-			hit_cnt += n_hit_result;
+			hit_cnt += scan_ret;
 		}
-		scan_ret = maat_scan_ipv6(g_doh_conf->maat,g_doh_conf->tables[TYPE_DST_ADDR].id, sapp_addr.v6->daddr,
-								 sapp_addr.v6->dest, 6, result+hit_cnt, MAX_SCAN_RESULT-hit_cnt,
-								 &n_hit_result, ctx->scan_mid);
-		if (scan_ret == MAAT_SCAN_HIT)
+		scan_ret = tfe_scan_ipv6_internal_addr(stream, result, ctx->scan_mid, hit_cnt, sapp_addr);
+		if (scan_ret > 0)
 		{
-			hit_cnt += n_hit_result;
+			hit_cnt += scan_ret;
 		}
 	}
 	// scan appid
-	int table_id=maat_get_table_id(g_doh_conf->maat, "APP_ID_DICT");
-	if(table_id < 0)
+	scan_ret = tfe_scan_app_id(result, ctx->scan_mid, hit_cnt, app_id, g_doh_conf->tables[TYPE_APPID].id);
+	if(scan_ret > 0)
 	{
-		return;
-	}
-	struct doh_app_id_dict *app_dict = (struct doh_app_id_dict *)maat_plugin_table_get_ex_data(g_doh_conf->maat, table_id, (const char *)&app_id, sizeof(long long));
-	if(app_dict!=NULL)
-	{
-		scan_ret = maat_scan_group(g_doh_conf->maat, g_doh_conf->tables[TYPE_APPID].id, &app_dict->group_id, 1, result + hit_cnt, MAX_SCAN_RESULT - hit_cnt, &n_hit_result,  ctx->scan_mid);
-		if(scan_ret==MAAT_SCAN_HIT)
-		{
-			TFE_LOG_INFO(g_doh_conf->local_logger, "Scan %s, Hit proto: %d scan ret: %d policy_id: %lld addr: %s",
-                     g_doh_conf->tables[TYPE_APPID].name, app_id, scan_ret, result[hit_cnt], stream->str_stream_info);
-        	hit_cnt += n_hit_result;
-		}
-		else
-		{
-        	TFE_LOG_INFO(g_doh_conf->local_logger, "Scan %s, NO hit proto: %d scan ret: %d addr: %s",
-        	             g_doh_conf->tables[TYPE_APPID].name, app_id, scan_ret, stream->str_stream_info);
-    	}
-		doh_app_dict_table_free_cb(0, (void **)&app_dict, 0, NULL);
+		hit_cnt += scan_ret;
 	}
 
 	// scan qname
@@ -422,6 +370,12 @@ static void doh_maat_scan(const struct tfe_stream *stream, const struct tfe_http
         TFE_LOG_INFO(g_doh_conf->local_logger, "Scan %s, NO hit domain: %s scan ret: %d addr: %s",
                      g_doh_conf->tables[TYPE_QNAME].name, qname, scan_ret, stream->str_stream_info);
     }
+	scan_ret = maat_scan_not_logic(g_doh_conf->maat, g_doh_conf->tables[TYPE_QNAME].id,
+								   result + hit_cnt, MAX_SCAN_RESULT - hit_cnt, &n_hit_result, ctx->scan_mid);
+	if (scan_ret == MAAT_SCAN_HIT)
+	{
+		hit_cnt += n_hit_result;
+	}
 
 	if (hit_cnt)
 	{
@@ -444,6 +398,8 @@ static int doh_maat_init(const char *profile, const char *section)
 	MESA_load_profile_string_def(profile, section, "table_qname", g_doh_conf->tables[TYPE_QNAME].name, TFE_STRING_MAX, "ATTR_DOH_QNAME");
 	MESA_load_profile_string_def(profile, section, "table_host", g_doh_conf->tables[TYPE_HOST].name, TFE_STRING_MAX, "ATTR_DOH_HOST");
 	MESA_load_profile_string_def(profile, section, "table_host_cat", g_doh_conf->tables[TYPE_HOST_CAT].name, TFE_STRING_MAX, "ATTR_DOH_HOST_CAT");
+	MESA_load_profile_string_def(profile, section, "table_internal_addr", g_doh_conf->tables[TYPE_INTERNAL_ADDR].name, TFE_STRING_MAX, "ATTR_INTERNAL_ADDR");
+	MESA_load_profile_string_def(profile, section, "table_external_addr", g_doh_conf->tables[TYPE_EXTERNAL_ADDR].name, TFE_STRING_MAX, "ATTR_EXTERNAL_ADDR");
 
 	for (int i = 0; i < TYPE_MAX; i++)
 	{
diff --git a/plugin/business/doh/src/logger.cpp b/plugin/business/doh/src/logger.cpp
index b9f986c..9cbaab4 100644
--- a/plugin/business/doh/src/logger.cpp
+++ b/plugin/business/doh/src/logger.cpp
@@ -20,6 +20,8 @@ enum _log_action //Bigger action number is prior.
     __LG_ACTION_MAX
 };
 
+#define get_time_ms(tv) ((long long)(tv.tv_sec) * 1000 + (long long)(tv.tv_usec) / 1000)
+
 static int get_rr_str2json(cJSON *object, dns_info_t *dns_info, int *dns_sec)
 {
     int i = 0;
@@ -308,7 +310,7 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
     char *log_payload = NULL;
     int kafka_status = 0;
     int send_cnt = 0;
-    time_t cur_time;
+    struct timeval cur_time;
     char src_ip_str[MAX(INET6_ADDRSTRLEN, INET_ADDRSTRLEN)] = {0};
     char dst_ip_str[MAX(INET6_ADDRSTRLEN, INET_ADDRSTRLEN)] = {0};
 
@@ -326,10 +328,10 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
     }
 
     common_obj = cJSON_CreateObject();
-    cur_time = time(NULL);
+    gettimeofday(&cur_time, NULL);
 
-    cJSON_AddNumberToObject(common_obj, "start_timestamp_ms", cur_time);
-    cJSON_AddNumberToObject(common_obj, "end_timestamp_ms", cur_time);
+    cJSON_AddNumberToObject(common_obj, "start_timestamp_ms", get_time_ms(cur_time));
+    cJSON_AddNumberToObject(common_obj, "end_timestamp_ms", get_time_ms(cur_time));
     cJSON_AddStringToObject(common_obj, "doh_version", app_proto[http->major_version]);
     cJSON_AddStringToObject(common_obj, "decoded_as", "DoH");
 
@@ -469,7 +471,7 @@ int doh_send_log(struct doh_conf *handle, const struct tfe_http_session *http, c
 
         TFE_LOG_DEBUG(handle->local_logger, "%s", log_payload);
 
-        kafka_status = tfe_kafka_logger_send(handle->kafka_logger, log_payload, strlen(log_payload));
+        kafka_status = tfe_kafka_logger_send(handle->kafka_logger, TOPIC_LOGGER, log_payload, strlen(log_payload));
         free(log_payload);
         cJSON_Delete(per_hit_obj);
         if (kafka_status < 0)
diff --git a/plugin/business/doh/src/pub.h b/plugin/business/doh/src/pub.h
index 6606312..6f830a2 100644
--- a/plugin/business/doh/src/pub.h
+++ b/plugin/business/doh/src/pub.h
@@ -36,6 +36,8 @@ enum table_type
 	TYPE_QNAME,
 	TYPE_HOST,
     TYPE_HOST_CAT,
+	TYPE_INTERNAL_ADDR,
+	TYPE_EXTERNAL_ADDR,
 	TYPE_MAX
 };