TSG-10254 eBPF支持二元组分流

TSG-10527 eBPF支持四元组分流
TSG-10369 KNI多线程在TAP模式下支持eBPF分流

7 files changed, 434 insertions, 161 deletions

diff --git a/entry/CMakeLists.txt b/entry/CMakeLists.txt
index d63c40b..f2883da 100644
--- a/entry/CMakeLists.txt
+++ b/entry/CMakeLists.txt
@@ -1,5 +1,11 @@
 set(CMAKE_INSTALL_PREFIX /home/mesasoft/sapp_run)
-add_library(kni SHARED src/kni_entry.cpp src/tfe_mgr.cpp src/kni_tun.cpp src/kni_pxy_tcp_option.cpp src/kni_dynamic_bypass.cpp)
+add_library(kni SHARED src/kni_entry.cpp src/tfe_mgr.cpp src/kni_tap_rss.cpp src/kni_pxy_tcp_option.cpp src/kni_dynamic_bypass.cpp)
 target_include_directories(kni PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include)
-target_link_libraries(kni common MESA_prof_load MESA_htable MESA_field_stat maatframe mrzcpd dabloom)
+
+if (SUPPORT_BPF)
+    target_link_libraries(kni common MESA_prof_load MESA_htable MESA_field_stat maatframe mrzcpd dabloom bpf)
+else()
+    target_link_libraries(kni common MESA_prof_load MESA_htable MESA_field_stat maatframe mrzcpd dabloom)
+endif()
+
 install(TARGETS kni LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/plug/business/kni COMPONENT LIBRARIES)
diff --git a/entry/include/kni_entry.h b/entry/include/kni_entry.h
index 1983caf..b5f9670 100644
--- a/entry/include/kni_entry.h
+++ b/entry/include/kni_entry.h
@@ -190,6 +190,9 @@ struct thread_tfe_cmsg_receiver_args{
 
 
 struct per_thread_handle{
+	int tap_fd;
+	int buff_size;
+	char *buff;
 	MESA_htable_handle tuple2stream_htable;
 	MESA_htable_handle traceid2sslinfo_htable;
 	struct expiry_dablooms_handle *dabloom_handle;
@@ -209,7 +212,7 @@ struct security_policy_shunt_tsg_diagnose{
 
 struct kni_handle{
 	struct kni_marsio_handle *marsio_handle;
-	struct kni_tun_handle *tun_handle;
+	struct bpf_ctx *tap_bpf_ctx;
 	struct kni_maat_handle *maat_handle;
 	struct kni_send_logger *send_logger;
 	MESA_htable_handle traceid2pme_htable;
diff --git a/entry/include/kni_tap_rss.h b/entry/include/kni_tap_rss.h
new file mode 100644
index 0000000..1421562
--- /dev/null
+++ b/entry/include/kni_tap_rss.h
@@ -0,0 +1,28 @@
+#ifndef _KNI_TAP_RSS_H_
+#define _KNI_TAP_RSS_H_
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#define TAP_RSS_LOG_TAG "TAP_RSS :"
+
+struct bpf_ctx;
+
+int kni_tap_get_bpf_prog_fd(struct bpf_ctx *ctx);
+
+struct bpf_ctx *kni_tap_global_load_rss_bpf(const char *bpf_obj_file, uint32_t bpf_queue_num, uint32_t bpf_hash_mode, uint32_t bpf_debug_log, void *logger);
+void kni_tap_global_unload_rss_bpf(struct bpf_ctx *ctx);
+
+int kni_tap_open_per_thread(const char *tap_dev, int tap_flags, int bpf_prog_fd, void *logger);
+void kni_tap_close_per_thread(int tap_fd);
+
+int kni_tap_read_per_thread(int tap_fd, char *buff, int buff_size, void *logger);
+int kni_tap_write_per_thread(int tap_fd, const char *data, int data_len, void *logger);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/entry/include/kni_tun.h b/entry/include/kni_tun.h
deleted file mode 100644
index 612a489..0000000
--- a/entry/include/kni_tun.h
+++ /dev/null
@@ -1,10 +0,0 @@
-enum kni_tun_mode{
-	KNI_TUN_MODE_BLOCK = 0,
-	KNI_TUN_MODE_NOBLOCK = 2,
-};
-
-struct kni_tun_handle;
-void kni_tun_destroy(struct kni_tun_handle *tun_handle);
-struct kni_tun_handle* kni_tun_init(char *tun_name, enum kni_tun_mode mode, void *logger);
-int kni_tun_write(struct kni_tun_handle *handle, char *buff, uint16_t buff_len);
-int kni_tun_read(struct kni_tun_handle *handle, char *buff, uint16_t buff_len);
\ No newline at end of file
diff --git a/entry/src/kni_entry.cpp b/entry/src/kni_entry.cpp
index 96e86d0..b0d581b 100644
--- a/entry/src/kni_entry.cpp
+++ b/entry/src/kni_entry.cpp
@@ -27,7 +27,8 @@ extern "C" {
 #ifdef __cplusplus
 }
 #endif
-#include "kni_tun.h"
+#include "kni_tap_rss.h"
+#include <linux/if_tun.h>
 #include <tsg/tsg_statistic.h>
 #include <MESA/stream_inc/stream_control.h>
 #include "kni_entry.h"
@@ -805,28 +806,34 @@ static int send_to_tfe_normal_mode(char *raw_data, uint16_t raw_len, int thread_
 	return 0;
 }
 
-static int send_to_tfe_tun_mode(char *raw_data, uint16_t raw_len, addr_type_t addr_type){
-	struct kni_tun_handle *handle = g_kni_handle->tun_handle;
-	char *dst_data = ALLOC(char, KNI_MTU);
-	add_ether_header(dst_data, raw_data, raw_len, addr_type);
-	int ret = kni_tun_write(handle, dst_data, raw_len + sizeof(struct ethhdr));
-	FREE(&dst_data);
-	if(ret < 0){
+static int send_to_tfe_tap_mode(char *raw_data, uint16_t raw_len, int thread_seq, addr_type_t addr_type, void *logger)
+{
+	int tap_fd = g_kni_handle->threads_handle[thread_seq].tap_fd;
+	char *send_buff = g_kni_handle->threads_handle[thread_seq].buff;
+
+	add_ether_header(send_buff, raw_data, raw_len, addr_type);
+	int ret = kni_tap_write_per_thread(tap_fd, send_buff, raw_len + sizeof(struct ethhdr), logger);
+	if (ret < 0)
+	{
 		return -1;
 	}
+
 	return 0;
 }
 
-static int send_to_tfe(char *raw_data, uint16_t raw_len, int thread_seq, int tfe_id, addr_type_t addr_type){
+static int send_to_tfe(char *raw_data, uint16_t raw_len, int thread_seq, int tfe_id, addr_type_t addr_type, void *logger)
+{
 	int mode = g_kni_handle->deploy_mode;
-	int ret;
-	if(mode == KNI_DEPLOY_MODE_TUN){
-		ret = send_to_tfe_tun_mode(raw_data, raw_len, addr_type);
-	}
-	else{
-		ret = send_to_tfe_normal_mode(raw_data, raw_len, thread_seq, tfe_id, addr_type);
+	switch (mode)
+	{
+	case KNI_DEPLOY_MODE_TAP:
+		return send_to_tfe_tap_mode(raw_data, raw_len, thread_seq, addr_type, logger);
+	case KNI_DEPLOY_MODE_NORMAL:
+		return send_to_tfe_normal_mode(raw_data, raw_len, thread_seq, tfe_id, addr_type);
+	default:
+		assert(0);
+		return -1;
 	}
-	return ret;
 }
 
 static void wrapped_kni_header_parse(const void *a_packet, struct pme_info *pmeinfo, struct pkt_info *pktinfo){
@@ -1062,10 +1069,6 @@ static int dabloom_add(struct pkt_info *pktinfo, int thread_seq){
 	return ret;
 }
 
-
-
-
-
 static struct session_attribute_label * kni_pull_session_attribute_results(struct streaminfo *a_stream,struct pme_info *pmeinfo)
 {
 	struct session_attribute_label *session_attribute = NULL;
@@ -1148,7 +1151,7 @@ static struct session_attribute_label * kni_pull_session_attribute_results(struc
 			}else{
 				KNI_LOG_DEBUG(logger, "share-session-attribute: destination ip location country is: %s,stream traceid = %s",session_attribute->server_location->country_full,pmeinfo->stream_traceid);			
 			}
-			if(session_attribute->server_location->province_full,pmeinfo->stream_traceid == NULL){
+			if(session_attribute->server_location->province_full == NULL){
 				KNI_LOG_DEBUG(logger, "share-session-attribute: destination ip location province is NULL,stream traceid = %s",pmeinfo->stream_traceid);
 			}else{
 				KNI_LOG_DEBUG(logger, "share-session-attribute: destination ip location province is: %s,stream traceid = %s",session_attribute->server_location->province_full,pmeinfo->stream_traceid);				
@@ -1438,7 +1441,7 @@ static int first_data_intercept(struct streaminfo *stream, struct pme_info *pmei
 	//Note: traceid2pme_add_fail, still work. no cmsg
 	traceid2pme_htable_add(pmeinfo);
 	//send to tfe
-	ret = send_to_tfe(buff, len, thread_seq, pmeinfo->tfe_id, pmeinfo->addr_type);
+	ret = send_to_tfe(buff, len, thread_seq, pmeinfo->tfe_id, pmeinfo->addr_type, logger);
 	if(ret < 0){
 		KNI_LOG_DEBUG(logger, "Intercept error: failed at send first packet to tfe%d, stream traceid = %s, stream addr = %s", 
 							pmeinfo->tfe_id, pmeinfo->stream_traceid, pmeinfo->stream_addr);
@@ -1578,7 +1581,7 @@ char next_data_intercept(struct pme_info *pmeinfo, const void *a_packet, struct
 		}
 	}
 
-	ret = send_to_tfe((char*)a_packet, len, thread_seq, pmeinfo->tfe_id, pmeinfo->addr_type);
+	ret = send_to_tfe((char*)a_packet, len, thread_seq, pmeinfo->tfe_id, pmeinfo->addr_type, logger);
 	if(ret < 0){
 		KNI_LOG_ERROR(logger, "Failed at send continue packet to tfe%d, stream traceid = %s, stream addr = %s", 
 						pmeinfo->tfe_id, pmeinfo->stream_traceid, pmeinfo->stream_addr);
@@ -1878,7 +1881,6 @@ int tuple2stream_htable_search(MESA_htable_handle handle, struct ethhdr *ether_h
 	tuple2stream_htable_value *value = NULL;
 	struct pkt_info pktinfo;
 	int reversed = 0, ret;
-	int key_size = 0;
 	char key_str[KNI_ADDR_MAX];
 	struct stream_tuple4_v6 key_v6;
 	struct stream_tuple4_v4 key_v4;
@@ -1973,21 +1975,28 @@ extern "C" char kni_polling_all_entry(const struct streaminfo *stream, void** pm
 			marsio_buff_free(g_kni_handle->marsio_handle->instance, rx_buffs, nr_recv, 0, 0);
 		}
 	}
-	//tun mode
-	else{
-		char buff[KNI_MTU];
-		int ret = kni_tun_read(g_kni_handle->tun_handle, buff, sizeof(buff));
-		if(ret < 0){
-			KNI_LOG_ERROR(logger, "Failed at read from tun");
-		}
-		else{
-			if(ret > 0){
-				struct ethhdr *ether_hdr = (struct ethhdr*)buff;
+	else if (g_kni_handle->deploy_mode == KNI_DEPLOY_MODE_TAP)
+	{
+		int tap_fd = g_kni_handle->threads_handle[thread_seq].tap_fd;
+		char *recv_buff = g_kni_handle->threads_handle[thread_seq].buff;
+		int recv_buff_size = g_kni_handle->threads_handle[thread_seq].buff_size;
+
+		// 只支持一个 TFE, 每次最多收 100 个包
+		for (int j = 0; j < 100; j++)
+		{
+			if (kni_tap_read_per_thread(tap_fd, recv_buff, recv_buff_size, logger) > 0)
+			{
+				struct ethhdr *ether_hdr = (struct ethhdr *)recv_buff;
 				tuple2stream_htable_search(tuple2stream_htable, ether_hdr, thread_seq);
 				flag = POLLING_STATE_WORK;
 			}
+			else
+			{
+				break;
+			}
 		}
 	}
+
 	return flag;
 }
 
@@ -2433,12 +2442,26 @@ error_out:
 	return NULL;
 }
 
+extern "C" void kni_destroy()
+{
+	if (g_kni_handle != NULL)
+	{
+		for (int i = 0; i < g_kni_handle->thread_count; i++)
+		{
+			kni_tap_close_per_thread(g_kni_handle->threads_handle[i].tap_fd);
+
+			if (g_kni_handle->threads_handle[i].buff)
+			{
+				free(g_kni_handle->threads_handle[i].buff);
+				g_kni_handle->threads_handle[i].buff = NULL;
+			}
+		}
+
+		kni_tap_global_unload_rss_bpf(g_kni_handle->tap_bpf_ctx);
 
-extern "C" void kni_destroy(){
-	if(g_kni_handle != NULL){
 		FREE(&g_kni_handle);
+		g_kni_handle = NULL;
 	}
-	g_kni_handle = NULL;
 }
 
 //eliminate_type: 0:FIFO; 1:LRU
@@ -2514,7 +2537,7 @@ extern "C" int kni_init(){
 	char *kni_git_verison = (char*)KNI_GIT_VERSION;
 	const char *profile = "./etc/kni/kni.conf";
 	const char *section = "global";
-
+	char tap_name[1024] = {0};
 	//init logger
 	char log_path[KNI_PATH_MAX] = "";
 	int tfe_node_count = 1;
@@ -2551,10 +2574,20 @@ extern "C" int kni_init(){
 
 	char deploy_mode[KNI_SYMBOL_MAX];
 	ret = MESA_load_profile_string_def(profile, section, "deploy_mode", deploy_mode, sizeof(deploy_mode), "normal");
-	g_kni_handle->deploy_mode = KNI_DEPLOY_MODE_NORMAL;
-	if(strcmp(deploy_mode, "tun") == 0){
-		g_kni_handle->deploy_mode = KNI_DEPLOY_MODE_TUN;
+	if (strcmp(deploy_mode, "tap") == 0)
+	{
+		g_kni_handle->deploy_mode = KNI_DEPLOY_MODE_TAP;
+	}
+	else if (strcmp(deploy_mode, "normal") == 0)
+	{
+		g_kni_handle->deploy_mode = KNI_DEPLOY_MODE_NORMAL;
+	}
+	else
+	{
+		KNI_LOG_ERROR(local_logger, "KNI Invalid deploy_mode: %s\n", deploy_mode);
+		goto error_out;
 	}
+	
 	if(g_kni_handle->deploy_mode == KNI_DEPLOY_MODE_NORMAL){
 		ret = MESA_load_profile_int_nodef(profile, section, "tfe_node_count", &tfe_node_count);
 		if(ret < 0){
@@ -2622,27 +2655,54 @@ extern "C" int kni_init(){
 			goto error_out;
 		}
 	}
-
-	//init tun
-	if(g_kni_handle->deploy_mode == KNI_DEPLOY_MODE_TUN){
-		if(g_kni_handle->thread_count != 1){
-			KNI_LOG_ERROR(local_logger, "Tun mode, thread count must be 1, while it's %d", g_kni_handle->thread_count);
+	else if (g_kni_handle->deploy_mode == KNI_DEPLOY_MODE_TAP)
+	{
+		if (tfe_node_count != 1)
+		{
+			KNI_LOG_ERROR(local_logger, TAP_RSS_LOG_TAG "under tap mode, only support one tfe node.");
 			goto error_out;
 		}
-		char tun_name[KNI_SYMBOL_MAX];
-		ret = MESA_load_profile_string_nodef(profile, section, "tun_name", tun_name, sizeof(tun_name));
-		if(ret < 0){
-			KNI_LOG_ERROR(local_logger, "MESA_prof_load: tun_name not set, profile = %s, section = %s", profile, section);
+
+		if (MESA_load_profile_string_nodef(profile, "tap", "tap_name", tap_name, sizeof(tap_name)) < 0)
+		{
+			KNI_LOG_ERROR(local_logger, TAP_RSS_LOG_TAG "under tap mode, tap_name is required.");
 			goto error_out;
 		}
-		KNI_LOG_ERROR(local_logger, "MESA_prof_load, [%s]:\n tun_name: %s", section, tun_name);
-		g_kni_handle->tun_handle = kni_tun_init(tun_name, KNI_TUN_MODE_NOBLOCK, local_logger);
-		if(g_kni_handle->tun_handle == NULL){
-			KNI_LOG_ERROR(local_logger, "Failed at init kni_tun");
-			goto error_out;
+
+		int tap_allow_mutilthread = 0;
+		MESA_load_profile_int_nodef(profile, "tap", "tap_allow_mutilthread", &tap_allow_mutilthread);
+		if (tap_allow_mutilthread)
+		{
+			uint32_t bpf_debug_log = 0;
+			uint32_t bpf_hash_mode = 2;
+			uint32_t bpf_queue_num = get_thread_count();
+			char bpf_obj[1024] = {0};
+			MESA_load_profile_int_nodef(profile, "tap", "bpf_debug_log", (int *)&bpf_debug_log);
+			MESA_load_profile_int_nodef(profile, "tap", "bpf_hash_mode", (int *)&bpf_hash_mode);
+			ret = MESA_load_profile_string_nodef(profile, "tap", "bpf_obj", bpf_obj, sizeof(bpf_obj));
+			if (ret < 0)
+			{
+				KNI_LOG_ERROR(local_logger, TAP_RSS_LOG_TAG "under tap mode, when enable tap_allow_mutilthread, bpf_obj is required.");
+				goto error_out;
+			}
+
+			g_kni_handle->tap_bpf_ctx = kni_tap_global_load_rss_bpf(bpf_obj, bpf_queue_num, bpf_hash_mode, bpf_debug_log, local_logger);
+			if (g_kni_handle->tap_bpf_ctx == NULL)
+			{
+				goto error_out;
+			}
+		}
+		else
+		{
+			if (get_thread_count() > 1)
+			{
+				KNI_LOG_ERROR(local_logger, TAP_RSS_LOG_TAG "under tap mode, when disable tap_allow_mutilthread, only support one work thread.");
+				goto error_out;
+			}
 		}
 	}
-	
+
+
 	//init_filedstat
 	fs_handle = fs_init(profile);
 	if(fs_handle == NULL){
@@ -2673,7 +2733,17 @@ extern "C" int kni_init(){
 			goto error_out;
 		}
 		g_kni_handle->threads_handle[i].tuple2stream_htable = tuple2stream_htable;
- 	}
+		if (g_kni_handle->deploy_mode == KNI_DEPLOY_MODE_TAP)
+		{
+			g_kni_handle->threads_handle[i].tap_fd = kni_tap_open_per_thread(tap_name, IFF_TAP | IFF_NO_PI | IFF_MULTI_QUEUE, kni_tap_get_bpf_prog_fd(g_kni_handle->tap_bpf_ctx), local_logger);
+			if (g_kni_handle->threads_handle[i].tap_fd == -1)
+			{
+				goto error_out;
+			}
+			g_kni_handle->threads_handle[i].buff_size = KNI_MTU;
+			g_kni_handle->threads_handle[i].buff = ALLOC(char, g_kni_handle->threads_handle[i].buff_size);
+		}
+	}
 	//init ssl dynamic bypass htable
 	ret = ssl_dynamic_bypass_htable_init(profile, local_logger);
 	if(ret < 0){
diff --git a/entry/src/kni_tap_rss.cpp b/entry/src/kni_tap_rss.cpp
new file mode 100644
index 0000000..667049c
--- /dev/null
+++ b/entry/src/kni_tap_rss.cpp
@@ -0,0 +1,267 @@
+
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <stdlib.h>
+#include <pthread.h>
+#include <stdarg.h>
+#include <arpa/inet.h>
+#include <netinet/ip.h>
+#include <netinet/ip6.h>
+#include <netinet/udp.h>
+
+#include <linux/if_tun.h>
+#include <signal.h>
+#include <assert.h>
+
+#if (SUPPORT_BPF)
+#include "../../bpf/bpf_conf_user.h"
+#include <bpf/bpf.h>
+#include <bpf/libbpf.h>
+#endif
+
+#include "kni_tap_rss.h"
+#include "kni_utils.h"
+
+#ifndef TUN_PATH
+#define TUN_PATH "/dev/net/tun"
+#endif
+
+struct bpf_ctx
+{
+    int bpf_prog_fd;
+    int bpf_map_fd;
+
+    char bpf_file[1024];
+#if (SUPPORT_BPF)
+    struct bpf_object *bpf_obj;
+    bpf_conf_t bpf_conf;
+#endif
+};
+
+int kni_tap_get_bpf_prog_fd(struct bpf_ctx *ctx)
+{
+    if (ctx)
+    {
+        return ctx->bpf_prog_fd;
+    }
+    else
+    {
+        return -1;
+    }
+}
+
+#if (SUPPORT_BPF)
+void kni_tap_global_unload_rss_bpf(struct bpf_ctx *ctx)
+{
+    if (ctx)
+    {
+        if (ctx->bpf_prog_fd > 0)
+        {
+            close(ctx->bpf_prog_fd);
+        }
+
+        if (ctx->bpf_obj)
+        {
+            bpf_object__close(ctx->bpf_obj);
+            ctx->bpf_obj = NULL;
+        }
+
+        free(ctx);
+        ctx = NULL;
+    }
+}
+#else
+void kni_tap_global_unload_rss_bpf(struct bpf_ctx *ctx)
+{
+}
+#endif
+
+/*
+ * bpf_queue_num        : worker thread number
+ * bpf_default_queue    : -1: for disable(only use for debug, rss to one queue)
+ * bpf_hash_mode        :  2: hash with src/dst addr
+ *                         4: hash with src/dst addr and src/dst port
+ * bpf_debug_log        :  0 for disable(only use for debug, printf bpf debug log)
+ */
+#if (SUPPORT_BPF)
+struct bpf_ctx *kni_tap_global_load_rss_bpf(const char *bpf_obj_file, uint32_t bpf_queue_num, uint32_t bpf_hash_mode, uint32_t bpf_debug_log, void *logger)
+{
+    struct bpf_ctx *ctx = (struct bpf_ctx *)calloc(1, sizeof(struct bpf_ctx));
+    strncpy(ctx->bpf_file, bpf_obj_file, strlen(bpf_obj_file));
+
+    bpf_conf_set_debug_log(&ctx->bpf_conf, bpf_debug_log);
+    bpf_conf_set_hash_mode(&ctx->bpf_conf, bpf_hash_mode);
+    bpf_conf_set_queue_num(&ctx->bpf_conf, bpf_queue_num);
+
+    if (bpf_prog_load(ctx->bpf_file, BPF_PROG_TYPE_SOCKET_FILTER, &ctx->bpf_obj, &ctx->bpf_prog_fd) < 0)
+    {
+        KNI_LOG_ERROR(logger, TAP_RSS_LOG_TAG "unable to load bpf object %s, aborting: %s\n", ctx->bpf_file, strerror(errno));
+        goto error;
+    }
+
+    if (bpf_conf_update_map(&ctx->bpf_conf, ctx->bpf_obj) == -1)
+    {
+        goto error;
+    }
+
+    return ctx;
+
+error:
+    kni_tap_global_unload_rss_bpf(ctx);
+
+    return NULL;
+}
+#else
+struct bpf_ctx *kni_tap_global_load_rss_bpf(const char *bpf_obj_file, uint32_t bpf_queue_num, uint32_t bpf_hash_mode, uint32_t bpf_debug_log, void *logger)
+{
+    KNI_LOG_ERROR(logger, TAP_RSS_LOG_TAG "not support bpf\n");
+    return NULL;
+}
+#endif
+
+int kni_tap_open_per_thread(const char *tap_dev, int tap_flags, int bpf_prog_fd, void *logger)
+{
+    int fd = -1;
+    int tap_fd = -1;
+    int nonblock_flags = -1;
+    struct ifreq ifr;
+
+    tap_fd = open(TUN_PATH, O_RDWR);
+    if (tap_fd == -1)
+    {
+        KNI_LOG_ERROR(logger, TAP_RSS_LOG_TAG "unable to open " TUN_PATH ", aborting: %s\n", strerror(errno));
+        return -1;
+    }
+
+    memset(&ifr, 0, sizeof(ifr));
+    ifr.ifr_flags = tap_flags;
+    strcpy(ifr.ifr_name, tap_dev);
+    if (ioctl(tap_fd, TUNSETIFF, &ifr) == -1)
+    {
+        KNI_LOG_ERROR(logger, TAP_RSS_LOG_TAG "unable to attach %s, aborting: %s\n", tap_dev, strerror(errno));
+        goto error;
+    }
+
+    /*
+     * The TUNSETPERSIST ioctl can be used to make the TUN/TAP interface persistent.
+     * In this mode, the interface won't be destroyed when the last process closes the associated /dev/net/tun file descriptor.
+     */
+    if (ioctl(tap_fd, TUNSETPERSIST, 1) == -1)
+    {
+        KNI_LOG_ERROR(logger, TAP_RSS_LOG_TAG "unable to set persist on %s, aborting: %s\n", tap_dev, strerror(errno));
+        goto error;
+    }
+
+#if (SUPPORT_BPF)
+    if (bpf_prog_fd > 0)
+    {
+        // Set bpf
+        if (ioctl(tap_fd, TUNSETSTEERINGEBPF, (void *)&bpf_prog_fd) == -1)
+        {
+            KNI_LOG_ERROR(logger, TAP_RSS_LOG_TAG "unable to set bpf on %s, aborting: %s\n", tap_dev, strerror(errno));
+            goto error;
+        }
+    }
+#endif
+
+    // Set nonblock
+    nonblock_flags = fcntl(tap_fd, F_GETFL);
+    if (nonblock_flags == -1)
+    {
+        KNI_LOG_ERROR(logger, TAP_RSS_LOG_TAG "unable to get nonblock flags on %s fd, aborting: %s\n", tap_dev, strerror(errno));
+        goto error;
+    }
+    nonblock_flags |= O_NONBLOCK;
+    if (fcntl(tap_fd, F_SETFL, nonblock_flags) == -1)
+    {
+        KNI_LOG_ERROR(logger, TAP_RSS_LOG_TAG "unable to set nonblock flags on %s fd, aborting: %s\n", tap_dev, strerror(errno));
+        goto error;
+    }
+
+    // Get MTU
+    fd = socket(PF_INET, SOCK_DGRAM, 0);
+    if (fd == -1)
+    {
+        KNI_LOG_ERROR(logger, TAP_RSS_LOG_TAG "unable to create socket, aborting: %s\n", strerror(errno));
+        goto error;
+    }
+
+    memset(&ifr, 0, sizeof(ifr));
+    strcpy(ifr.ifr_name, tap_dev);
+    if (ioctl(fd, SIOCGIFMTU, &ifr) < 0)
+    {
+        KNI_LOG_ERROR(logger, TAP_RSS_LOG_TAG "unable to get MTU on %s, aborting: %s\n", tap_dev, strerror(errno));
+        goto error;
+    }
+
+    // Set eth up
+    if (ioctl(fd, SIOCGIFFLAGS, &ifr) == -1)
+    {
+        KNI_LOG_ERROR(logger, TAP_RSS_LOG_TAG "unable to get link status on %s, aborting: %s\n", tap_dev, strerror(errno));
+        goto error;
+    }
+
+    if ((ifr.ifr_flags & IFF_UP) == 0)
+    {
+        ifr.ifr_flags |= IFF_UP;
+        if (ioctl(fd, SIOCSIFFLAGS, &ifr) < 0)
+        {
+            KNI_LOG_ERROR(logger, TAP_RSS_LOG_TAG "unable to set link status on %s, aborting: %s\n", tap_dev, strerror(errno));
+            goto error;
+        }
+    }
+
+    KNI_LOG_INFO(logger, TAP_RSS_LOG_TAG "using tap device %s with MTU %d\n", tap_dev, ifr.ifr_mtu);
+    close(fd);
+
+    return tap_fd;
+
+error:
+
+    if (fd > 0)
+    {
+        close(fd);
+        fd = -1;
+    }
+
+    if (tap_fd > 0)
+    {
+        close(tap_fd);
+        tap_fd = -1;
+    }
+
+    return -1;
+}
+
+void kni_tap_close_per_thread(int tap_fd)
+{
+    if (tap_fd > 0)
+    {
+        close(tap_fd);
+    }
+}
+
+int kni_tap_read_per_thread(int tap_fd, char *buff, int buff_size, void *logger)
+{
+    int ret = read(tap_fd, buff, buff_size);
+    if (ret < 0)
+    {
+        if (errno != EWOULDBLOCK && errno != EAGAIN)
+        {
+            KNI_LOG_ERROR(logger, TAP_RSS_LOG_TAG "unable to read data from tapfd %d, aborting: %s\n", tap_fd, strerror(errno));
+        }
+    }
+
+    return ret;
+}
+
+int kni_tap_write_per_thread(int tap_fd, const char *data, int data_len, void *logger)
+{
+    int ret = write(tap_fd, data, data_len);
+    if (ret != data_len)
+    {
+        KNI_LOG_ERROR(logger, TAP_RSS_LOG_TAG "need send %dB, only send %dB, aborting: %s\n", data_len, ret, strerror(errno));
+    }
+
+    return ret;
+}
\ No newline at end of file
diff --git a/entry/src/kni_tun.cpp b/entry/src/kni_tun.cpp
deleted file mode 100644
index 284b157..0000000
--- a/entry/src/kni_tun.cpp
+++ /dev/null
@@ -1,91 +0,0 @@
-#include "kni_utils.h"
-#include <sys/ioctl.h>
-#include <fcntl.h>
-#include <net/if.h>
-#include <linux/if_tun.h> 
-#include <kni_tun.h>
-
-struct kni_tun_handle{
-    int fd;
-    void *logger;
-    enum kni_tun_mode mode;
-};
-
-void kni_tun_destroy(struct kni_tun_handle *tun_handle){
-    if(tun_handle != NULL){
-        close(tun_handle->fd);
-        FREE(&tun_handle);
-    }
-}
-
-struct kni_tun_handle* kni_tun_init(char *tun_name, enum kni_tun_mode mode, void *logger){
-    struct kni_tun_handle *tun_handle = ALLOC(struct kni_tun_handle, 1);
-    tun_handle->logger = logger;
-    char *tun_path = (char*)"/dev/net/tun";
-    struct ifreq ifr;
-    memset(&ifr, 0, sizeof(ifr));
-    int fd = -1, ret;
-	ifr.ifr_flags = IFF_TAP | IFF_NO_PI | IFF_ONE_QUEUE;
-    if(tun_name == NULL || *tun_name == '\0'){
-        KNI_LOG_ERROR(logger, "Tap device name is NULL");
-        goto error_out;
-    }
-    strncpy(ifr.ifr_name, tun_name, IFNAMSIZ);
-    fd = open(tun_path, O_RDWR);
-    if(fd < 0){
-        KNI_LOG_ERROR(logger, "Failed at open file, filename = %s, errno = %d(%s)", tun_path, errno, strerror(errno));
-        goto error_out;
-    }
-    tun_handle->fd = fd;
-    tun_handle->mode = KNI_TUN_MODE_NOBLOCK;
-    if(tun_handle->mode == KNI_TUN_MODE_NOBLOCK){
-        int flags = fcntl(fd, F_GETFL, 0);
-        ret = fcntl(fd, F_SETFL, flags | O_NONBLOCK);
-        if(ret < 0){
-            KNI_LOG_ERROR(logger ,"Failed at fcntl, fd = %d, flags = F_GETFL | O_NONBLOCK, errno = %d(%s)", errno, strerror(errno));
-            goto error_out;
-        }
-    }
-    ret = ioctl(tun_handle->fd, TUNSETIFF, (void *)&ifr);
-	if(ret < 0){
-		KNI_LOG_ERROR(logger ,"Failed at ioctl, fd = %d, errno = %d(%s)", errno, strerror(errno));
-		goto error_out;
-	}
-    return tun_handle;
-
-error_out:
-    kni_tun_destroy(tun_handle);
-    return NULL;
-}
-
-
-int kni_tun_write(struct kni_tun_handle *handle, char *buff, uint16_t buff_len){
-	uint16_t ret = write(handle->fd, buff, buff_len);
-	if(ret < 0){
-        KNI_LOG_ERROR(handle->logger, "Failed at write, errno = %d(%s)", errno, strerror(errno));
-        return -1;
-	}
-    if(ret < buff_len){
-        KNI_LOG_ERROR(handle->logger, "Failed at write: need send %dB, only send %dB", buff_len, ret);
-        return -1;
-    }
-	return 0;
-}
-
-/*
-*   >= 0 : read data length
-*   = -1 : error
-*/
-int kni_tun_read(struct kni_tun_handle *handle, char *buff, uint16_t buff_len){
-    int ret = 0;
-	ret = read(handle->fd, buff, buff_len);
-	if(ret < 0){
-        if(ret == -1 && handle->mode == KNI_TUN_MODE_NOBLOCK){
-            if(errno == EWOULDBLOCK || errno == EAGAIN){
-                return 0;
-            }
-        }
-        KNI_LOG_ERROR(handle->logger, "Failed at read, error = %d(%s)", errno, strerror(errno));
-	}
-	return ret;
-}
\ No newline at end of file