#1 实现基于ASAN的大页内存保护模式v4.3.0-20181207

* app初始化时对所有大页面增加保护，当应用调用MARSIO API时，对需要读写的区域解保护，以此实现对越界大页面读写的检测；
* 由于rte_hash使用的大页面内存太分散，不便于保护，改用MESA_htable实现ARP表；
* 增加大页面保护编译选项。

16 files changed, 615 insertions, 158 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index fee5006..750f5d5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -10,14 +10,37 @@ include(Version)
 
 # EXTRA Flags
 # GNU99 and C++11 Support
-set(EXTRA_C_FLAGS "${EXTRA_C_FLAGS} -std=gnu99 -m64 -march=native")
-set(EXTRA_CXX_FLAGS "${EXTRA_CXX_FLAGS} -std=gnu++11 -m64 -march=native" )
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${EXTRA_C_FLAGS}")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${EXTRA_CXX_FLAGS}")
+option(ENABLE_WARNING_ALL "Enable all optional warnings which are desirable for normal code" TRUE)
+option(ENABLE_SANITIZE_ADDRESS "Enable AddressSanitizer" FALSE)
+option(ENABLE_SANITIZE_THREAD "Enable ThreadSanitizer" FALSE)
+option(ENABLE_VNODE_CHECK_THREAD_SAFE "Enable concurrent write thread-safe check for VNODE." FALSE)
+
+set(CMAKE_C_FLAGS "-std=gnu99 -m64 -march=native")
+set(CMAKE_CXX_FLAGS "-std=gnu++11 -m64 -march=native" )
+
+if(ENABLE_WARNING_ALL)
+	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -fPIC")
+	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -fPIC")
+endif()
+
+if(ENABLE_SANITIZE_ADDRESS)
+	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address -fno-omit-frame-pointer")
+	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer")
+	set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan")
+	set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan")
+elseif(ENABLE_SANITIZE_THREAD)
+	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=thread -fno-omit-frame-pointer")
+	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread -fno-omit-frame-pointer")
+	set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan")
+	set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan")
+endif()
+
+if(ENABLE_VNODE_CHECK_THREAD_SAFE)
+	add_definitions(-DVNODE_CHECK_THREAD_SAFE)
+endif()
 
 # Default Install Destination Directory
 set(CMAKE_INSTALL_PREFIX /opt/mrzcpd)
-add_definitions(-fPIC -Wall)
 add_definitions(-D_GNU_SOURCE -D__STDC_LIMIT_MACROS)
 
 include_directories(include/external)
@@ -33,8 +56,7 @@ set(MR_INSTALL_LDCONFDIR /etc/ld.so.conf.d/)
 set(MR_INSTALL_SYSUNITCONFDIR /etc/sysconfig/)
 set(MR_INSTALL_PROFILEDIR /etc/profile.d/)
 
-exec_program(pkg-config ARGS systemd --variable=systemdsystemunitdir
-	OUTPUT_VARIABLE MR_INSTALL_SYSUNITDIR)
+exec_program(pkg-config ARGS systemd --variable=systemdsystemunitdir OUTPUT_VARIABLE MR_INSTALL_SYSUNITDIR)
 
 add_subdirectory(${CMAKE_SOURCE_DIR}/support)
 add_subdirectory(${CMAKE_SOURCE_DIR}/include)
diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt
index e61a7d8..ff14044 100644
--- a/app/CMakeLists.txt
+++ b/app/CMakeLists.txt
@@ -13,7 +13,7 @@ add_library(marsio SHARED src/marsio.c src/arp.c src/icmp.c src/neigh.c src/rawi
 set_target_properties(marsio PROPERTIES VERSION ${MARSIO_VERSION_MAJOR}.${MARSIO_VERSION_MINOR})
 set_target_properties(marsio PROPERTIES SOVERSION ${MARSIO_VERSION_MAJOR})
 set_target_properties(marsio PROPERTIES LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_LIST_DIR}/src/version.map")
-target_link_libraries(marsio PRIVATE ${DPDK_LIBRARY} infra rt pthread dl MESA_prof_load_static ${SYSTEMD_LIBRARIES})
+target_link_libraries(marsio PRIVATE ${DPDK_LIBRARY} infra rt pthread dl MESA_prof_load_static MESA_htable_static ${SYSTEMD_LIBRARIES})
 target_include_directories(marsio INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include/")
 target_include_directories(marsio INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include/external")
 target_include_directories(marsio INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include/internal")
diff --git a/app/include/mrapp.h b/app/include/mrapp.h
index 7d42dc2..55bb24f 100644
--- a/app/include/mrapp.h
+++ b/app/include/mrapp.h
@@ -35,8 +35,6 @@ struct mrapp_stat
     uint64_t shared_men_free;
 };
 
-
-
 /* 用户设备描述符 */
 struct mr_vdev
 {
@@ -120,6 +118,8 @@ struct mr_instance
     thread_id_t to_suppose_tid;
     /* RECV ALL的状态暂存 */
     unsigned int recv_all_state[MR_SID_MAX];
+    /* 基于ASAN的内存保护模式 */
+    unsigned int memory_protect_with_asan;
 };
 
 /* EAL环境是否初始化 */
diff --git a/app/include/neigh.h b/app/include/neigh.h
index 880010c..d10b870 100644
--- a/app/include/neigh.h
+++ b/app/include/neigh.h
@@ -8,6 +8,7 @@ extern "C" {
 #include <rte_rwlock.h>
 #include <vdev_define.h>
 #include <cJSON.h>
+#include <MESA_htable.h>
 
 struct neighbour;
 TAILQ_HEAD(neighbour_list, neighbour);
@@ -15,9 +16,9 @@ TAILQ_HEAD(neighbour_list, neighbour);
 struct neighbour_manager
 {
 	/* 读写锁 */
-	rte_rwlock_t rwlock;
+	rte_spinlock_t lock;
 	/* 查询哈希表 */
-	struct rte_hash * table;
+	MESA_htable_handle table;
 	/* 待查询表项队列 */
 	struct neighbour_list in_none_list;
 	/* 待查询表项队列长度（不受rwlock的保护） */
diff --git a/app/src/arp.c b/app/src/arp.c
index 9bac957..b67f18a 100644
--- a/app/src/arp.c
+++ b/app/src/arp.c
@@ -14,6 +14,7 @@
 #include <mrapp.h>
 #include <netinet/in.h>
 #include <arp.h>
+#include <protect.h>
 
 static struct ether_addr broadcast_hwaddr = { { 0xFF,0xFF,0xFF,0xFF,0xFF,0xFF } };
 
@@ -69,11 +70,11 @@ static void arp_request_entry(struct neighbour_manager * neigh_manager, struct v
 	if (!__local_inaddr_filter(vdi->vdev, d_in_addr))
 		goto done;
 
-	struct rte_mbuf * reply_mbuf = rte_pktmbuf_alloc(vdi->direct_pool);
+	struct rte_mbuf * reply_mbuf = PROTECT_rte_pktmbuf_alloc(vdi->direct_pool);
 	if (unlikely(reply_mbuf == NULL)) goto done;
 	
 	// 构造以太网头
-	struct ether_hdr * ether_hdr = (struct ether_hdr *)rte_pktmbuf_append(
+	struct ether_hdr * ether_hdr = (struct ether_hdr *) PROTECT_rte_pktmbuf_append(
 		reply_mbuf, sizeof(struct ether_hdr));
 
 	ether_addr_copy(&vdi->vdev->ether_addr, &ether_hdr->s_addr);
@@ -81,7 +82,7 @@ static void arp_request_entry(struct neighbour_manager * neigh_manager, struct v
 	ether_hdr->ether_type = ntohs(ETHER_TYPE_ARP);
 
 	// 构造ARP应答
-	struct arp_hdr * reply_arp_hdr = (struct arp_hdr *)rte_pktmbuf_append(
+	struct arp_hdr * reply_arp_hdr = (struct arp_hdr *) PROTECT_rte_pktmbuf_append(
 		reply_mbuf, sizeof(struct arp_hdr));
 
 	rte_memcpy(reply_arp_hdr, arp_header, sizeof(struct arp_hdr));
@@ -144,11 +145,11 @@ int arp_request_send(struct vdev_instance* vdi, queue_id_t qid, struct in_addr i
 	struct ether_addr * dst_hwaddr = &broadcast_hwaddr;
 
 	struct in_addr in_addr_src = dev_info->in_addr;
-	struct rte_mbuf * req_mbuf = rte_pktmbuf_alloc(vdi->direct_pool);
+	struct rte_mbuf * req_mbuf = PROTECT_rte_pktmbuf_alloc(vdi->direct_pool);
 	if (unlikely(req_mbuf == NULL)) return -ENOBUFS;
 
 	// 构造以太网头
-	struct ether_hdr * ether_hdr = (struct ether_hdr *)rte_pktmbuf_append(
+	struct ether_hdr * ether_hdr = (struct ether_hdr *) PROTECT_rte_pktmbuf_append(
 		req_mbuf, sizeof(struct ether_hdr));
 
 	ether_addr_copy(src_hwaddr, &ether_hdr->s_addr);
@@ -156,7 +157,7 @@ int arp_request_send(struct vdev_instance* vdi, queue_id_t qid, struct in_addr i
 	ether_hdr->ether_type = htons(ETHER_TYPE_ARP);
 
 	// 构造ARP请求报文
-	struct arp_hdr * arp_hdr = (struct arp_hdr *)rte_pktmbuf_append(
+	struct arp_hdr * arp_hdr = (struct arp_hdr *) PROTECT_rte_pktmbuf_append(
 		req_mbuf, sizeof(struct arp_hdr));
 
 	arp_hdr->arp_hrd = htons(ARP_HRD_ETHER);
diff --git a/app/src/icmp.c b/app/src/icmp.c
index 70f00c7..81b6b8c 100644
--- a/app/src/icmp.c
+++ b/app/src/icmp.c
@@ -2,12 +2,15 @@
 #include <mrapp.h>
 #include <rte_icmp.h>
 #include <rte_ip.h>
+#include <protect.h>
 
 static void icmp_echo_request_entry(struct vdev_instance* vdi, queue_id_t sid,
 	struct rte_mbuf * mbuf, struct in_addr s_in_addr, struct in_addr d_in_addr)
 {
     /* 将原来的报文复制一份 */
-    struct rte_mbuf * mbuf_cloned = rte_pktmbuf_alloc(vdi->direct_pool);
+    struct rte_mbuf * mbuf_cloned = PROTECT_rte_pktmbuf_alloc(vdi->direct_pool);
+    PROTECT_rte_mbuf_unpoison(mbuf_cloned);
+
     char * _pkt_data = rte_pktmbuf_append(mbuf_cloned, rte_pktmbuf_data_len(mbuf));
     rte_memcpy(_pkt_data, rte_pktmbuf_mtod(mbuf, char *), rte_pktmbuf_data_len(mbuf));
 
diff --git a/app/src/marsio.c b/app/src/marsio.c
index 211b341..da69baa 100644
--- a/app/src/marsio.c
+++ b/app/src/marsio.c
@@ -6,6 +6,7 @@
 #include <rte_malloc.h>
 #include <rte_version.h>
 #include <rte_string_fns.h>
+#include <rte_spinlock.h>
 
 #include <mrapp.h>
 #include <marsio.h>
@@ -18,12 +19,16 @@
 #include <cJSON.h>
 #include <libgen.h>
 #include <arp.h>
+#include <rte_acl_osdep.h>
+#include <protect.h>
 
 #define MR_LIB_MAX_EAL_ARGC			128
 
 unsigned int g_logger_to_stdout = 1;
 unsigned int g_logger_level = LOG_DEBUG;
 unsigned int g_eal_inited = 0;
+unsigned int g_in_protect_mode = 0;
+rte_spinlock_t g_in_protect_lock = RTE_SPINLOCK_INITIALIZER;
 
 __thread struct mr_thread_info thread_info;
 
@@ -67,6 +72,48 @@ static void __write_arg(char * eal_argv[], unsigned int * eal_argc,
 	do {__write_arg(eal_argv, &eal_argc, MR_LIB_MAX_EAL_ARGC, x);}		\
 	while(0)
 
+
+void __mrapp_mem_protect_unlock_mempool_cb(struct rte_mempool * mp, void *arg)
+{
+    PROTECT_rte_mempool_unpoison(mp);
+    MR_INFO("Unlock MEMPOOL %s: %p, local cache is %p", mp->name, mp, mp->local_cache);
+}
+
+/* 基于ASAN的内存保护模式
+ * 该保护模式开启后，共享大页内存将标记为不可达，对共享内存的访问将被ASAN探测并记录 */
+static void mrapp_mem_protect_with_asan_init(struct mr_instance * instance)
+{
+    MESA_load_profile_uint_def(instance->app_cfgfile_path, "protect", "enable",
+        &instance->memory_protect_with_asan, 0);
+
+    if(instance->memory_protect_with_asan)
+    {
+#ifndef __SANITIZE_ADDRESS__
+        MR_WARNING("Memory Protection with ASAN is not supported for the APP is not compiled with address_sanitizer.");
+        instance->memory_protect_with_asan = 0; return;
+#endif
+    }
+    else { return; }
+
+    MR_INFO("Memory Protection with ASAN is enabled.");
+    g_in_protect_mode = 1;
+    struct rte_config *cfg = rte_eal_get_configuration();
+    const struct rte_mem_config * mcfg = cfg->mem_config;
+
+    for(int i = 0; i < RTE_MAX_MEMSEG; i++)
+    {
+        if(mcfg->memseg[i].addr == NULL) break;
+        void * memseg_addr = mcfg->memseg[i].addr;
+        size_t memseg_len = mcfg->memseg[i].len;
+
+        MR_ASAN_POISON_MEMORY_REGION(memseg_addr, memseg_len);
+        MR_INFO("MEMSEG %d is protected by ASAN: addr = %p, len = %zu", i, memseg_addr, memseg_len);
+    }
+
+    /* 解锁所有的mempool */
+    rte_mempool_walk(__mrapp_mem_protect_unlock_mempool_cb, NULL);
+}
+
 /* EAL环境初始化 */
 static void mrapp_eal_init(struct mr_instance * instance)
 {
@@ -627,6 +674,17 @@ static int __ctrlplane_conn_close_handler(struct ctrlmsg_handler * ct_hand,
     exit(EXIT_FAILURE);
 }
 
+static void __open_device_unposion(struct vdev_instance * vdi)
+{
+    MR_ASAN_UNPOISON_MEMORY_REGION(vdi, sizeof(struct vdev_instance));
+    MR_ASAN_UNPOISON_MEMORY_REGION(vdi->vdev, sizeof(struct vdev));
+
+    vnode_mirror_unpoison_cons(vdi->vnode_rx_cons);
+    vnode_mirror_unpoison_prod(vdi->vnode_tx_prod);
+    vnode_mirror_unpoison_prod(vdi->vnode_ftx_prod);
+    vnode_mirror_unpoison_prod(vdi->vnode_ltx_prod);
+}
+
 static int __open_device_response_handler(struct ctrlmsg_handler * ct_hand,
     struct ctrlmsg_conn * ct_conn, struct ctrl_msg_header * msg, void * arg)
 {
@@ -649,6 +707,12 @@ static int __open_device_response_handler(struct ctrlmsg_handler * ct_hand,
     mr_vdev->instance = instance;
     strncpy(mr_vdev->devsym, (char *)rep_msg->devsym, MR_SYMBOL_MAX);
 
+    /* VDI使用了大页面共享内存，在保护模式下撤除对它的保护 */
+    if(instance->memory_protect_with_asan)
+    {
+        __open_device_unposion(mr_vdev->vdi);
+    }
+
     /* 启用了IP地址，处理ARP请求和ICMP请求 */
     struct vdev * vdev = mr_vdev->vdi->vdev;
     if (vdev->in_addr.s_addr != 0 && vdev->in_mask.s_addr != 0)
@@ -866,6 +930,9 @@ int marsio_init(struct mr_instance * instance, const char * appsym)
         goto err;
     }
 
+    /* 初始化ASAN内存保护模式 */
+    mrapp_mem_protect_with_asan_init(instance);
+
     pthread_t pid_ctrlplane_thread;
     int ret = pthread_create(&pid_ctrlplane_thread, NULL, mrapp_ctrlplane_thread, instance);
     if (ret < 0)
diff --git a/app/src/mrb.c b/app/src/mrb.c
index 912e9e2..4a1c64d 100644
--- a/app/src/mrb.c
+++ b/app/src/mrb.c
@@ -7,11 +7,19 @@
 #include <marsio.h>
 #include <mrapp.h>
 #include <mrb_define.h>
+#include <protect.h>
 
 void * marsio_buff_ctrlzone(marsio_buff_t *mr_buff, uint8_t id)
 {
+	PROTECT_rte_mbuf_unpoison_meta((struct rte_mbuf *)mr_buff);
+	struct mrb_zone_idx* cz = mrbuf_cz(mr_buff, id);
 	assert(id < mrbuf_cz_num(mr_buff));
-	return mrbuf_cz_data(mr_buff, id);
+
+	void * ctrl_addr = mrbuf_cz_data(mr_buff, id);
+	unsigned int ctrl_len = cz->size;
+
+	MR_ASAN_UNPOISON_MEMORY_REGION(ctrl_addr, ctrl_len);
+	return ctrl_addr;
 }
 
 void * marsio_buff_ctrlzone_data(marsio_buff_t *mr_buff, uint8_t id, uint8_t *size)
@@ -24,25 +32,30 @@ void * marsio_buff_ctrlzone_data(marsio_buff_t *mr_buff, uint8_t id, uint8_t *si
 
 void marsio_buff_ctrlzone_set(marsio_buff_t *mr_buff, uint8_t id, void* ptr_data, uint8_t size)
 {
+	PROTECT_rte_mbuf_unpoison_meta((struct rte_mbuf *)mr_buff);
 	struct mrb_zone_idx* cz = mrbuf_cz(mr_buff, id);
 	RTE_SET_USED(cz);
 
 	assert(id < mrbuf_cz_num(mr_buff));
 	assert(size <= cz->size);
+
+	MR_ASAN_UNPOISON_MEMORY_REGION(mrbuf_cz_data(mr_buff, id), cz->size);
 	memcpy(mrbuf_cz_data(mr_buff, id), ptr_data, size);
-	return;
 }
 
 void * mr_buffer_ctrlzone(struct rte_mbuf * mr_buff, uint8_t id)
 {
     assert(id < mrbuf_cz_num(mr_buff));
+	struct mrb_zone_idx* cz = mrbuf_cz(mr_buff, id);
+
+	MR_ASAN_UNPOISON_MEMORY_REGION(mrbuf_cz_data(mr_buff, id), cz->size);
     return mrbuf_cz_data(mr_buff, id);
 }
 
 int marsio_buff_malloc_device(struct mr_vdev * vdev, marsio_buff_t *marsio_buff[], 
     unsigned int nr_mbufs, int socket_id, int thread_id)
 {
-    int ret = rte_pktmbuf_alloc_bulk(vdev->vdi->direct_pool, 
+    int ret = PROTECT_rte_pktmbuf_alloc_bulk(vdev->vdi->direct_pool,
 			(struct rte_mbuf **)marsio_buff, nr_mbufs);
 
 #ifndef NDEBUG
@@ -82,8 +95,9 @@ void marsio_buff_free(struct mr_instance * instance, marsio_buff_t *marsio_buff[
 {
 	for (int i = 0; i < nr_mbufs; i++)
 	{
+		PROTECT_rte_mbuf_unpoison_meta((struct rte_mbuf *)marsio_buff[i]);
 		mrbuf_priv(marsio_buff[i])->next = NULL;
-		rte_pktmbuf_free((struct rte_mbuf *)marsio_buff[i]);
+		PROTECT_rte_pktmbuf_free((struct rte_mbuf *)marsio_buff[i]);
 	}
 
     thread_id = thread_info.thread_id;
@@ -93,36 +107,57 @@ void marsio_buff_free(struct mr_instance * instance, marsio_buff_t *marsio_buff[
 
 void marsio_buff_append_pkt(marsio_buff_t *head, marsio_buff_t *next)
 {
+	PROTECT_rte_mbuf_unpoison_meta((struct rte_mbuf *)head);
+	PROTECT_rte_mbuf_unpoison_meta((struct rte_mbuf *)next);
+
 	marsio_buff_t *p = head;
 	while (mrbuf_priv(p)->next != NULL)
 	{
 		p = mrbuf_priv(p)->next;
 	}
 	mrbuf_priv(p)->next = next;
+
+	PROTECT_rte_mbuf_poison_meta((struct rte_mbuf *)head);
+	PROTECT_rte_mbuf_poison_meta((struct rte_mbuf *)next);
 }
 
 void marsio_buff_append_seg(marsio_buff_t *head, marsio_buff_t *next)
 {
+	PROTECT_rte_mbuf_unpoison_meta((struct rte_mbuf *)head);
+	PROTECT_rte_mbuf_unpoison_meta((struct rte_mbuf *)next);
+
 	rte_pktmbuf_chain((struct rte_mbuf *)head, (struct rte_mbuf *)next);
+
+	PROTECT_rte_mbuf_poison_meta((struct rte_mbuf *)head);
+	PROTECT_rte_mbuf_poison_meta((struct rte_mbuf *)next);
 }
 
 marsio_buff_t *marsio_buff_getnext_seg(marsio_buff_t *m)
 {
+	PROTECT_rte_mbuf_unpoison_meta((struct rte_mbuf *)m);
 	return (marsio_buff_t *)(((struct rte_mbuf*)m)->next);
+	PROTECT_rte_mbuf_poison_meta((struct rte_mbuf *)m);
 }
 
 marsio_buff_t *marsio_buff_getnext_pkt(marsio_buff_t *m)
 {
+	PROTECT_rte_mbuf_unpoison_meta((struct rte_mbuf *)m);
 	return mrbuf_priv(m)->next;
+	PROTECT_rte_mbuf_poison_meta((struct rte_mbuf *)m);
 }
 
 void marsio_buff_chain_pkt(marsio_buff_t * pkt, marsio_buff_t * next)
 {
+	PROTECT_rte_mbuf_unpoison_meta((struct rte_mbuf *)pkt);
+	PROTECT_rte_mbuf_unpoison_meta((struct rte_mbuf *)next);
     mrbuf_priv(pkt)->next = next;
+	PROTECT_rte_mbuf_poison_meta((struct rte_mbuf *)pkt);
+	PROTECT_rte_mbuf_poison_meta((struct rte_mbuf *)next);
 }
 
 void marsio_buff_reset(marsio_buff_t *m)
 {
+	PROTECT_rte_mbuf_unpoison_meta(m);
 	struct mrb_priv_zone* priv_zone = mrbuf_priv(m);
 	struct mrb_zone_idx* cz_first = NULL, *cz_last = NULL;
     
@@ -141,22 +176,25 @@ void marsio_buff_reset(marsio_buff_t *m)
 		cz_last = mrbuf_cz(m, mrbuf_cz_num(m) - 1);
 		memset(mrbuf_cz_data(m, 0), 0, cz_last->offset + cz_last->size - cz_first->offset);
 	}
-	return;
+
+	PROTECT_rte_mbuf_poison_meta(m);
 }
 
 char *marsio_buff_mtod(marsio_buff_t *m)
 {
+	PROTECT_rte_mbuf_unpoison_meta((struct rte_mbuf *)m);
+	PROTECT_rte_mbuf_unpoison_data((struct rte_mbuf *)m);
 	return rte_pktmbuf_mtod((struct rte_mbuf *)m, char *);
 }
 
 uint32_t marsio_buff_buflen(marsio_buff_t *m)
 {
-	return rte_pktmbuf_pkt_len((struct rte_mbuf *)m);
+	return PROTECT_rte_pktmbuf_pkt_len((struct rte_mbuf *)m);
 }
 
 uint32_t marsio_buff_datalen(marsio_buff_t *m)
 {
-	return rte_pktmbuf_data_len((struct rte_mbuf *)m);
+	return PROTECT_rte_pktmbuf_data_len((struct rte_mbuf *)m);
 }
 
 struct rte_mbuf * __pktmbuf_alloc(struct mr_instance * instance, int socket_id, int thread_id)
@@ -334,36 +372,36 @@ char *marsio_buff_prepend(marsio_buff_t *m, uint16_t len)
 {
 	/* 引用计数检查，修改报文时，不应该被其他的地方引用 */
 	struct rte_mbuf * __mbuf = (struct rte_mbuf *)m;
-	if (unlikely(rte_mbuf_refcnt_read(__mbuf) != 1)) return NULL;
+	if (unlikely(PROTECT_rte_mbuf_refcnt_read(__mbuf) != 1)) return NULL;
 
-	return rte_pktmbuf_prepend((struct rte_mbuf *)m, len);
+	return PROTECT_rte_pktmbuf_prepend((struct rte_mbuf *)m, len);
 }
 
 char *marsio_buff_append(marsio_buff_t *m, uint16_t len)
 {
 	/* 引用计数检查，修改报文时，不应该被其他的地方引用 */
 	struct rte_mbuf * __mbuf = (struct rte_mbuf *)m;
-	if (unlikely(rte_mbuf_refcnt_read(__mbuf) != 1)) return NULL;
+	if (unlikely(PROTECT_rte_mbuf_refcnt_read(__mbuf) != 1)) return NULL;
 
-	return rte_pktmbuf_append((struct rte_mbuf *)m, len);
+	return PROTECT_rte_pktmbuf_append((struct rte_mbuf *)m, len);
 }
 
 char *marsio_buff_adj(marsio_buff_t *m, uint16_t len)
 {
 	/* 引用计数检查，修改报文时，不应该被其他的地方引用 */
 	struct rte_mbuf * __mbuf = (struct rte_mbuf *)m;
-	if (unlikely(rte_mbuf_refcnt_read(__mbuf) != 1)) return NULL;
+	if (unlikely(PROTECT_rte_mbuf_refcnt_read(__mbuf) != 1)) return NULL;
 
-	return rte_pktmbuf_adj((struct rte_mbuf *)m, len);
+	return PROTECT_rte_pktmbuf_adj((struct rte_mbuf *)m, len);
 }
 
 int marsio_buff_trim(marsio_buff_t *m, uint16_t len)
 {
 	/* 引用计数检查，修改报文时，不应该被其他的地方引用 */
 	struct rte_mbuf * __mbuf = (struct rte_mbuf *)m;
-	if (unlikely(rte_mbuf_refcnt_read(__mbuf) != 1)) return -1;
+	if (unlikely(PROTECT_rte_mbuf_refcnt_read(__mbuf) != 1)) return -1;
 
-	return rte_pktmbuf_trim((struct rte_mbuf *)m, len);
+	return PROTECT_rte_pktmbuf_trim((struct rte_mbuf *)m, len);
 }
 
 char * marsio_buff_offset_set(marsio_buff_t *m, off_t offset, 
@@ -526,8 +564,9 @@ void marsio_pktmbuf_dump(FILE *f, const marsio_buff_t *m, unsigned dump_len)
 
 uint64_t marsio_buff_get_timestamp(marsio_buff_t *m)
 {
+	if(g_in_protect_mode) return 0;
     struct mrb_priv_zone * priv_zone = mrbuf_priv(m);
-    return priv_zone->ts.ts_app_rx_or_alloc.tv_nsec;
+    return (uint64_t)priv_zone->ts.ts_app_rx_or_alloc.tv_nsec;
 }
 
 void marsio_buff_set_timestamp(marsio_buff_t *m, uint64_t timestamp)
diff --git a/app/src/neigh.c b/app/src/neigh.c
index bb7aa30..bc36103 100644
--- a/app/src/neigh.c
+++ b/app/src/neigh.c
@@ -24,6 +24,8 @@
 #include <cJSON.h>
 #include <arpa/inet.h>
 
+#include <MESA_htable.h>
+
 enum neigh_state
 {
 	NEIGH_IN_NONE = 0,
@@ -68,48 +70,43 @@ struct neighbour
 int neighbour_mamanger_init(struct neighbour_manager * object,
 	const char * symbol, unsigned int max_entries, unsigned short t_timeout, unsigned int t_arp_send)
 {
-	int ret = 0;
+	const static unsigned int __opt_disable = 0;
 
-	char tbsymbol[MR_SYMBOL_MAX];
-	snprintf(tbsymbol, sizeof(tbsymbol), "NEIGH_%s", symbol);
-
-	struct rte_hash_parameters hash_params =
+	object->table = MESA_htable_born();
+	if(unlikely(object->table == NULL))
 	{
-		.name = tbsymbol,
-		.entries = max_entries,
-		.key_len = sizeof(struct in_addr),
-		.hash_func = NULL,
-		.hash_func_init_val = 0,
-        .socket_id = SOCKET_ID_ANY
-	};
-
-    /* 检查是否存在以前的哈希表 */
-    struct rte_hash * exist_hash = rte_hash_find_existing(tbsymbol);
-    if (exist_hash != NULL) rte_hash_free(exist_hash);
-
-	object->table = rte_hash_create(&hash_params);
-	if (unlikely(object->table == NULL))
+		MR_INFO("Cannot born MESA_htable(), the ret is NULL. ");
+		goto errout;
+	}
+
+	/* 不使用MESA_htable的线程安全特性，在外层用自旋锁保证线程安全
+	 * TODO: 使用MESA_htable的线程安全特性，并增加超时淘汰等特性 */
+	MESA_htable_set_opt(object->table, MHO_THREAD_SAFE, (void *)&__opt_disable, sizeof(__opt_disable));
+    MESA_htable_set_opt(object->table, MHO_HASH_MAX_ELEMENT_NUM, &max_entries, sizeof(max_entries));
+
+    int ret = MESA_htable_mature(object->table);
+    if(ret < 0)
 	{
-		MR_ERROR("Cannot create hash table for %s : %s", symbol, __str_rte_errno());
-		ret = -1; goto errout;
+		MR_INFO("Cannot mature MESA_htable(), the ret is %d. ", ret);
+		goto errout;
 	}
 
 	TAILQ_INIT(&object->in_none_list);
 	rte_atomic16_init(&object->in_none_list_len);
-	rte_rwlock_init(&object->rwlock);
+	rte_spinlock_init(&object->lock);
 
-    object->t_arp_send = t_arp_send;
+    object->t_arp_send = (unsigned short)t_arp_send;
     object->t_timeout = t_timeout;
 	return 0;
 
 errout:
-	if (object->table != NULL) rte_hash_free(object->table);
-	return ret;
+	if (object->table != NULL) MESA_htable_destroy(object->table, NULL);
+	return -1;
 }
 
 int neighbour_mamanger_deinit(struct neighbour_manager * object)
 {
-	rte_hash_free(object->table);
+	MESA_htable_destroy(object->table, NULL);
 	return 0;
 }
 
@@ -126,12 +123,8 @@ static struct neighbour * __neigh_create_and_join_hash(struct neighbour_manager
 	struct in_addr in_addr, struct ether_addr * eth_addr, 
 	struct vdev_instance * vdi, unsigned short en_permanent)
 {
-	struct neighbour * neigh = rte_zmalloc(NULL, sizeof(struct neighbour), 0);
-	if (unlikely(neigh == NULL))
-	{
-        MR_ERROR("Cannot create neigh structure : %s", __str_rte_errno());
-		return NULL;
-	}
+	struct neighbour * neigh = malloc(sizeof(struct neighbour));
+	memset(neigh, 0, sizeof(struct neighbour));
 
 	neigh->in_addr = in_addr;
     neigh->vdi = vdi;
@@ -151,18 +144,17 @@ static struct neighbour * __neigh_create_and_join_hash(struct neighbour_manager
 		neigh->en_permanent = 1;
 	}
 
-	hash_sig_t hash = rte_hash_crc(&in_addr, sizeof(in_addr), 0);
-	int ret = rte_hash_add_key_with_hash_data(object->table, &in_addr, hash, neigh);
+	int ret = MESA_htable_add(object->table, (const unsigned char *)&in_addr, sizeof(in_addr), neigh);
 	if (unlikely(ret < 0))
 	{
-		MR_ERROR("Insert neigh to hash table failed : %s", __str_rte_errno());
+		MR_ERROR("Insert neigh to hash table failed, ret = %d", ret);
 		goto errout;
 	}
 
 	return neigh;
 	
 errout:
-	if (neigh) rte_free(neigh);
+	if (neigh) free(neigh);
 	return NULL;
 }
 
@@ -186,12 +178,9 @@ static void __neigh_update(struct neighbour * neigh, struct in_addr in_addr,
 int neigh_create_or_update(struct neighbour_manager * object, struct in_addr in_addr, 
 	struct ether_addr * eth_addr, struct vdev_instance * vdi, unsigned short en_permanent)
 {
-	rte_rwlock_write_lock(&object->rwlock);
-	struct neighbour * neigh = NULL;
-	hash_sig_t hash = rte_hash_crc(&in_addr, sizeof(in_addr), 0);
-
-	int ret = rte_hash_lookup_with_hash_data(object->table, &in_addr, hash, (void **)&neigh);
-	if (ret < 0)
+	rte_spinlock_lock(&object->lock);
+	struct neighbour * neigh = MESA_htable_search(object->table, (const unsigned char *)&in_addr, sizeof(in_addr));
+	if (unlikely(neigh == NULL))
 	{
 		neigh = __neigh_create_and_join_hash(object, in_addr, eth_addr, vdi, en_permanent);
 	}
@@ -200,6 +189,7 @@ int neigh_create_or_update(struct neighbour_manager * object, struct in_addr in_
 		__neigh_update(neigh, in_addr, eth_addr, vdi, en_permanent);
 	}
 
+	int ret = 0;
     if (unlikely(neigh == NULL)) { ret = -1; goto exit; }
 
     time_t now_time = time(NULL);
@@ -212,18 +202,15 @@ int neigh_create_or_update(struct neighbour_manager * object, struct in_addr in_
 	ret = 0; goto exit;
 	
 exit:
-	rte_rwlock_write_unlock(&object->rwlock);
+	rte_spinlock_unlock(&object->lock);
 	return ret;
 }
 
 int neigh_delete(struct neighbour_manager * object, struct in_addr in_addr)
 {
-	rte_rwlock_write_lock(&object->rwlock);
-	struct neighbour * neigh = NULL;
-
-	hash_sig_t hash = rte_hash_crc(&in_addr, sizeof(in_addr), 0);
-	int ret = rte_hash_lookup_with_hash_data(object->table, &in_addr, hash, (void **)&neigh);
-	if (ret < 0) goto exit;
+	rte_spinlock_lock(&object->lock);
+	struct neighbour * neigh = MESA_htable_search(object->table, (const unsigned char *)&in_addr, sizeof(in_addr));
+	if (neigh == NULL) goto exit;
 
 	// 当处于IN_NONE状态时，从IN_NONE列表中移除
 	if (neigh->state == NEIGH_IN_NONE)
@@ -232,26 +219,23 @@ int neigh_delete(struct neighbour_manager * object, struct in_addr in_addr)
 		rte_atomic16_dec(&object->in_none_list_len);
 	}
 
-	rte_hash_del_key_with_hash(object->table, &in_addr, hash);
-	rte_free(neigh);
-	ret = 0; goto exit;
+	MESA_htable_del(object->table, (const unsigned char *)&in_addr, sizeof(in_addr), NULL);
+	return 0;
 
 exit:
-	rte_rwlock_write_unlock(&object->rwlock);
-	return ret;
+	rte_spinlock_unlock(&object->lock);
+	return -1;
 }
 
 int neigh_query(struct neighbour_manager * object, struct in_addr in_addr,
 	struct ether_addr * out_ether_addr, struct vdev_instance ** vdi)
 {
-	struct neighbour * neigh = NULL;
+	rte_spinlock_lock(&object->lock);
+	struct neighbour * neigh = MESA_htable_search(object->table, (const unsigned char *)&in_addr, sizeof(in_addr));
 
-	rte_rwlock_read_lock(&object->rwlock);
-	hash_sig_t hash = rte_hash_crc(&in_addr, sizeof(in_addr), 0);
-	int ret = rte_hash_lookup_with_hash_data(object->table, &in_addr, hash, (void **)&neigh);
-	
 	// 没查到
-	if (ret < 0) goto exit;
+	int ret = 0;
+	if (neigh == NULL) goto exit;
 
 	// 以下状态信息不全，无法对外提供查询
 	if (neigh->state == NEIGH_IN_NONE ||
@@ -266,7 +250,7 @@ int neigh_query(struct neighbour_manager * object, struct in_addr in_addr,
 	ret = 0; goto exit;
 
 exit:
-	rte_rwlock_read_unlock(&object->rwlock);
+	rte_spinlock_unlock(&object->lock);
 	return ret;
 }
 
@@ -274,7 +258,7 @@ static void __neigh_handle_in_no_queue(struct neighbour_manager * object, queue_
 {
 	if (rte_atomic16_read(&object->in_none_list_len) == 0) return;
 	
-	rte_rwlock_write_lock(&object->rwlock);
+	rte_spinlock_lock(&object->lock);
 	struct neighbour * neigh_iter;
 	
 	// 对每一个在In-None列表中的Neigh，发ARP请求。
@@ -290,7 +274,7 @@ static void __neigh_handle_in_no_queue(struct neighbour_manager * object, queue_
 
 	// In-None列表应当为空，数量计数应当为0
 	assert(rte_atomic16_read(&object->in_none_list_len) == 0);
-	rte_rwlock_write_unlock(&object->rwlock);
+	rte_spinlock_unlock(&object->lock);
 }
 
 void neighbour_manager_loop_entry(struct neighbour_manager * object, queue_id_t qid)
@@ -298,34 +282,36 @@ void neighbour_manager_loop_entry(struct neighbour_manager * object, queue_id_t
 	__neigh_handle_in_no_queue(object, qid);
 }
 
+void __neighbour_manager_monit_iterate(const uchar * key, uint size, void * data, void *user)
+{
+	struct neighbour * iter_neigh = (struct neighbour *)data;
+	struct cJSON * j_root = (struct cJSON *)user;
+	assert(iter_neigh != NULL && j_root != NULL);
+
+	char str_in_addr[MR_STRING_MAX] = { 0 };
+	inet_ntop(AF_INET, &iter_neigh->in_addr, str_in_addr, INET_ADDRSTRLEN);
+
+	char str_ether_addr[MR_STRING_MAX] = { 0 };
+	ether_format_addr(str_ether_addr, sizeof(str_ether_addr), &iter_neigh->eth_addr);
+
+	cJSON * j_neigh = cJSON_CreateObject();
+	cJSON_AddStringToObject(j_neigh, "InAddress", str_in_addr);
+	cJSON_AddStringToObject(j_neigh, "HWAddress", str_ether_addr);
+	cJSON_AddStringToObject(j_neigh, "Interface", iter_neigh->vdi->vdev->symbol);
+	cJSON_AddStringToObject(j_neigh, "State", str_neigh_state[iter_neigh->state]);
+
+	cJSON_AddItemToArray(j_root, j_neigh);
+	(void)key;
+	(void)size;
+}
+
 cJSON * neighbour_manager_monit(struct neighbour_manager * object)
 {
 	struct cJSON * j_root = cJSON_CreateArray();
 	MR_VERIFY_MALLOC(j_root);
 
-	struct in_addr iter_in_addr;
-	struct neighbour * iter_neigh;
-	uint32_t iterate = 0;
-
-	rte_rwlock_read_lock(&object->rwlock);
-	while (rte_hash_iterate(object->table, (const void **)&iter_in_addr, 
-		(void **)&iter_neigh, &iterate) >= 0)
-	{
-		char str_in_addr[MR_STRING_MAX] = { 0 };
-		inet_ntop(AF_INET, &iter_neigh->in_addr, str_in_addr, INET_ADDRSTRLEN);
-
-		char str_ether_addr[MR_STRING_MAX] = { 0 };
-		ether_format_addr(str_ether_addr, sizeof(str_ether_addr), &iter_neigh->eth_addr);
-		
-		cJSON * j_neigh = cJSON_CreateObject();
-		cJSON_AddStringToObject(j_neigh, "InAddress", str_in_addr);
-		cJSON_AddStringToObject(j_neigh, "HWAddress", str_ether_addr);
-		cJSON_AddStringToObject(j_neigh, "Interface", iter_neigh->vdi->vdev->symbol);
-		cJSON_AddStringToObject(j_neigh, "State", str_neigh_state[iter_neigh->state]);
-
-		cJSON_AddItemToArray(j_root, j_neigh);
-	}
-	
-	rte_rwlock_read_unlock(&object->rwlock);
+	rte_spinlock_lock(&object->lock);
+	MESA_htable_iterate(object->table, __neighbour_manager_monit_iterate, j_root);
+	rte_spinlock_unlock(&object->lock);
 	return j_root;
 }
\ No newline at end of file
diff --git a/app/src/rawio.c b/app/src/rawio.c
index 1423bc3..c30b410 100644
--- a/app/src/rawio.c
+++ b/app/src/rawio.c
@@ -8,8 +8,9 @@
 #include <arp.h>
 #include <icmp.h>
 #include <sendpath.h>
+#include <protect.h>
 
-static inline int __packet_total_len(struct rte_mbuf * mbufs[], unsigned int nr_mbufs)
+static inline unsigned int __packet_total_len(struct rte_mbuf * mbufs[], unsigned int nr_mbufs)
 {
     unsigned int total_len = 0;
     for (int i = 0; i < nr_mbufs; i++) total_len += rte_pktmbuf_pkt_len(mbufs[i]);
@@ -19,6 +20,8 @@ static inline int __packet_total_len(struct rte_mbuf * mbufs[], unsigned int nr_
 int mrapp_packet_fast_send_burst(struct vdev_instance * vdi, queue_id_t qid,
     struct rte_mbuf * mbufs[], int nr_mbufs)
 {
+    PROTECT_rte_mbuf_unpoison_bulk(mbufs, nr_mbufs);
+
     hash_t mbufs_hash[MR_BURST_MAX];
     for (int i = 0; i < nr_mbufs; i++)
     {
@@ -33,6 +36,7 @@ int mrapp_packet_fast_send_burst(struct vdev_instance * vdi, queue_id_t qid,
 
     //TODO: 锁换一个位置
     static rte_spinlock_t __f_fast_lock = { 0 };
+    PROTECT_rte_mbuf_poison_bulk(mbufs, nr_mbufs);
 
     rte_spinlock_lock(&__f_fast_lock);
     int ret = vnode_mirror_enqueue_bulk(vdi->vnode_ltx_prod, qid, mbufs, mbufs_hash, nr_mbufs);
@@ -51,6 +55,8 @@ int marsio_recv_burst(struct mr_vdev * vdev, queue_id_t qid, marsio_buff_t * mbu
     int ret = vnode_mirror_dequeue_burst(vdi->vnode_rx_cons, qid, __mbufs, nr_mbufs);
     if (unlikely(ret == 0)) goto out;
 
+    PROTECT_rte_mbuf_unpoison_bulk(__mbufs, ret);
+
     /* ARP */
     if (vdev->en_arp) arp_entry(vdev->instance, vdi, qid, __mbufs, ret);
     /* ICMP */
@@ -69,8 +75,12 @@ int marsio_recv_burst(struct mr_vdev * vdev, queue_id_t qid, marsio_buff_t * mbu
 
     /* BPF Dumper */
     if (unlikely(vdev->bpf_dumper != NULL))
+    {
         bpf_dumper_write(vdev->bpf_dumper, __mbufs, ret);
 
+    }
+
+    PROTECT_rte_mbuf_poison_bulk(__mbufs, ret);
 out:
     return ret;
 }
@@ -122,6 +132,8 @@ err:
 int marsio_send_burst(struct mr_sendpath * sendpath, queue_id_t sid, marsio_buff_t * mbufs[], int nr_mbufs)
 {
     struct rte_mbuf ** __mbufs = (struct rte_mbuf **)mbufs;
+    PROTECT_rte_mbuf_unpoison_bulk(__mbufs, nr_mbufs);
+
     for (int i = 0; i < nr_mbufs; i++) __rte_mbuf_sanity_check(mbufs[i], i);
 
     if (__send_burst_packet_construct(sendpath, sid, mbufs, nr_mbufs) < 0)
@@ -135,8 +147,6 @@ int marsio_send_burst(struct mr_sendpath * sendpath, queue_id_t sid, marsio_buff
     if (unlikely(sendpath->vdev->bpf_dumper != NULL))
         bpf_dumper_write(sendpath->vdev->bpf_dumper, __mbufs, nr_mbufs);
 
-    vnode_mirror_enqueue_bulk(sendpath->target_vdi->vnode_tx_prod, sid, __mbufs, hash, nr_mbufs);
-
     /* 线程运行情况统计 */
     if (thread_info.instance != NULL)
     {
@@ -145,10 +155,14 @@ int marsio_send_burst(struct mr_sendpath * sendpath, queue_id_t sid, marsio_buff
         thread_info.instance->stat[tid].packet_send_length = __packet_total_len(__mbufs, nr_mbufs);
     }
 
+    /* 提交到队列前，应用不再具有访问权限，对mbuf进行保护 */
+    PROTECT_rte_mbuf_poison_bulk(__mbufs, nr_mbufs);
+
+    /* 写入队列 */
+    vnode_mirror_enqueue_bulk(sendpath->target_vdi->vnode_tx_prod, sid, __mbufs, hash, nr_mbufs);
     return RT_SUCCESS;
 
 err:
-
     /* 线程丢包情况统计 */
     if (thread_info.instance != NULL)
     {
@@ -156,7 +170,7 @@ err:
         thread_info.instance->stat[tid].packet_send_drop += nr_mbufs;
     }
 
-    for (int i = 0; i < nr_mbufs; i++) rte_pktmbuf_free(mbufs[i]);
+    for (int i = 0; i < nr_mbufs; i++) PROTECT_rte_pktmbuf_free(mbufs[i]);
     return RT_ERR;
 }
 
@@ -164,6 +178,8 @@ int marsio_send_burst_with_options(struct mr_sendpath * sendpath, queue_id_t sid
     marsio_buff_t * mbufs[], int nr_mbufs, uint16_t options)
 {
     struct rte_mbuf ** __mbufs = (struct rte_mbuf **)mbufs;
+    PROTECT_rte_mbuf_unpoison_bulk(__mbufs, nr_mbufs);
+
     assert(sid < sendpath->target_vdi->nr_txstream);
 
     for (int i = 0; i < nr_mbufs; i++)
@@ -197,6 +213,17 @@ int marsio_send_burst_with_options(struct mr_sendpath * sendpath, queue_id_t sid
     if (unlikely(sendpath->vdev->bpf_dumper != NULL))
         bpf_dumper_write(sendpath->vdev->bpf_dumper, __mbufs, nr_mbufs);
 
+    /* 线程运行情况统计 */
+    if (thread_info.instance != NULL)
+    {
+        thread_id_t tid = thread_info.thread_id;
+        thread_info.instance->stat[tid].packet_send_count += nr_mbufs;
+        thread_info.instance->stat[tid].packet_send_length = __packet_total_len(__mbufs, nr_mbufs);
+    }
+
+    /* 提交到队列前，应用不再具有访问权限，对mbuf进行保护 */
+    PROTECT_rte_mbuf_poison_bulk(__mbufs, nr_mbufs);
+
     if (options & MARSIO_SEND_OPT_FAST)
     {
         vnode_mirror_enqueue_bulk(sendpath->target_vdi->vnode_ftx_prod, sid,
@@ -208,14 +235,6 @@ int marsio_send_burst_with_options(struct mr_sendpath * sendpath, queue_id_t sid
             __mbufs, hash, nr_mbufs);
     }
 
-    /* 线程运行情况统计 */
-    if (thread_info.instance != NULL)
-    {
-        thread_id_t tid = thread_info.thread_id;
-        thread_info.instance->stat[tid].packet_send_count += nr_mbufs;
-        thread_info.instance->stat[tid].packet_send_length = __packet_total_len(__mbufs, nr_mbufs);
-    }
-
     return RT_SUCCESS;
 
 err:
diff --git a/include/internal/protect.h b/include/internal/protect.h
new file mode 100644
index 0000000..ec23b4c
--- /dev/null
+++ b/include/internal/protect.h
@@ -0,0 +1,232 @@
+#pragma once

+

+#include <rte_mempool.h>

+#include <rte_mbuf.h>

+#include <rte_spinlock.h>

+

+/* 全局asan开关 */

+extern unsigned int g_in_protect_mode;

+

+/* asan_poison和asan_unpoison非线程安全，

+ * 设置一个全局锁，保证不会同时执行asan_posion和asan_unpoison操作 */

+extern rte_spinlock_t g_in_protect_lock;

+

+/* TODO: 这些定义应通过包含GCC的头文件来实现，不应通过这种方式 */

+#if defined(__SANITIZE_ADDRESS__)

+void __asan_poison_memory_region(void const volatile *addr, size_t size);

+void __asan_unpoison_memory_region(void const volatile *addr, size_t size);

+#endif

+

+#if defined(__SANITIZE_ADDRESS__)

+#define MR_ASAN_POISON_MEMORY_REGION(addr, size)   do {                     \

+if(g_in_protect_mode)                                                       \

+{                                                                           \

+    rte_spinlock_lock(&g_in_protect_lock);                                  \

+    __asan_poison_memory_region((addr), (size));                            \

+    rte_spinlock_unlock(&g_in_protect_lock);                                \

+}} while(0)

+

+#define MR_ASAN_UNPOISON_MEMORY_REGION(addr, size) do {                     \

+if(g_in_protect_mode)                                                       \

+{                                                                           \

+    rte_spinlock_lock(&g_in_protect_lock);                                  \

+    __asan_unpoison_memory_region((addr), (size));                          \

+    rte_spinlock_unlock(&g_in_protect_lock);                                \

+}} while(0)

+

+#else

+#define MR_ASAN_POISON_MEMORY_REGION(addr, size) ((void)(addr), (void)(size))

+#define MR_ASAN_UNPOISON_MEMORY_REGION(addr, size) ((void)(addr), (void)(size))

+#endif

+

+static inline void PROTECT_rte_mempool_unpoison(struct rte_mempool *pool)

+{

+    MR_ASAN_UNPOISON_MEMORY_REGION(pool, sizeof(struct rte_mempool));

+    MR_ASAN_UNPOISON_MEMORY_REGION(pool, MEMPOOL_HEADER_SIZE(pool, pool->cache_size));

+}

+

+static inline void PROTECT_rte_mbuf_unpoison_meta(struct rte_mbuf *mbuf)

+{

+    MR_ASAN_UNPOISON_MEMORY_REGION(mbuf, sizeof(struct rte_mbuf));

+    MR_ASAN_UNPOISON_MEMORY_REGION((const char *)mbuf + sizeof(struct rte_mbuf), mbuf->priv_size);

+}

+

+static inline void PROTECT_rte_mbuf_poison_meta(struct rte_mbuf *mbuf)

+{

+    MR_ASAN_POISON_MEMORY_REGION((const char *)mbuf + sizeof(struct rte_mbuf), mbuf->priv_size);

+    MR_ASAN_POISON_MEMORY_REGION(mbuf, sizeof(struct rte_mbuf));

+}

+

+static inline void PROTECT_rte_mbuf_poison_buff(struct rte_mbuf *mbuf)

+{

+    MR_ASAN_POISON_MEMORY_REGION(mbuf->buf_addr, mbuf->buf_len);

+}

+

+static inline void PROTECT_rte_mbuf_unpoison_buff(struct rte_mbuf *mbuf)

+{

+    MR_ASAN_UNPOISON_MEMORY_REGION(mbuf->buf_addr, mbuf->buf_len);

+}

+

+static inline void PROTECT_rte_mbuf_poison(struct rte_mbuf * mbuf)

+{

+    PROTECT_rte_mbuf_poison_buff(mbuf);

+    PROTECT_rte_mbuf_poison_meta(mbuf);

+}

+

+static inline void PROTECT_rte_mbuf_unpoison(struct rte_mbuf * mbuf)

+{

+    PROTECT_rte_mbuf_unpoison_meta(mbuf);

+    PROTECT_rte_mbuf_unpoison_buff(mbuf);

+}

+

+static inline void PROTECT_rte_mbuf_poison_bulk(struct rte_mbuf * mbufs[], int nr_mbufs)

+{

+    for(unsigned int i = 0; i < nr_mbufs; i++) PROTECT_rte_mbuf_poison(mbufs[i]);

+}

+

+static inline void PROTECT_rte_mbuf_unpoison_bulk(struct rte_mbuf * mbufs[], int nr_mbufs)

+{

+    for(unsigned int i = 0; i < nr_mbufs; i++) PROTECT_rte_mbuf_unpoison(mbufs[i]);

+}

+

+static inline void PROTECT_rte_mbuf_unpoison_data(struct rte_mbuf *mbuf)

+{

+    /* Poison all buff area, because some area may be unpoison before. */

+    PROTECT_rte_mbuf_poison_buff(mbuf);

+

+    /* Unpoison the area we need to access */

+    const char * data_ptr = rte_pktmbuf_mtod(mbuf, const char *);

+    size_t data_len = rte_pktmbuf_data_len(mbuf);

+    MR_ASAN_UNPOISON_MEMORY_REGION(data_ptr, data_len);

+}

+

+/* 从DPDK17.05中移植rte_pktmbuf_alloc

+ * 该函数在申请mbuf的同时还要修改mbuf中的元数据，此时由于还没有拿到mbuf的指针，

+ * 也就无法去保护，函数功能无法实现。现移植，在拿到mbuf的指针后立即去保护，继续后面的初始化流程 */

+static inline struct rte_mbuf * __PROTECT_PORT_rte_mbuf_raw_alloc(struct rte_mempool *mp)

+{

+    struct rte_mbuf *m;

+    void *mb = NULL;

+

+    if (rte_mempool_get(mp, &mb) < 0)

+        return NULL;

+

+    m = (struct rte_mbuf *)mb;

+    PROTECT_rte_mbuf_unpoison_meta(m);

+    MBUF_RAW_ALLOC_CHECK(m);

+    return m;

+}

+

+static inline struct rte_mbuf * __PROTECT_PORT_rte_pktmbuf_alloc(struct rte_mempool *mp)

+{

+    struct rte_mbuf *m;

+    if ((m = __PROTECT_PORT_rte_mbuf_raw_alloc(mp)) != NULL)

+        rte_pktmbuf_reset(m);

+    return m;

+}

+

+/* 移植完毕，以后DPDK版本升级时，要格外注意以上的实现是否发生了变化 */

+

+static inline struct rte_mbuf * PROTECT_rte_pktmbuf_alloc(struct rte_mempool *mp)

+{

+#ifndef __SANITIZE_ADDRESS__

+    return rte_pktmbuf_alloc(mp);

+#else

+    struct rte_mbuf * m = __PROTECT_PORT_rte_pktmbuf_alloc(mp);

+    PROTECT_rte_mbuf_poison_meta(m);

+    return m;

+#endif

+}

+

+static inline void PROTECT_rte_pktmbuf_free(struct rte_mbuf *mbuf)

+{

+    PROTECT_rte_mbuf_unpoison_meta(mbuf);

+    rte_pktmbuf_free(mbuf);

+    /* TODO: 应重新对释放的mbuf区域进行保护，需要在释放前计算mbuf的指针，长度，再执行保护 */

+}

+

+static inline int PROTECT_rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, struct rte_mbuf **mbufs, unsigned count)

+{

+#ifndef __SANITIZE_ADDRESS__

+    return rte_pktmbuf_alloc_bulk(pool, mbufs, count);

+#else

+

+    unsigned int alloc_count;

+    for(alloc_count = 0; alloc_count < count; alloc_count++)

+    {

+        struct rte_mbuf * m = PROTECT_rte_pktmbuf_alloc(pool);

+        if(unlikely(m == NULL)) goto errout;

+        mbufs[alloc_count] = m;

+    }

+

+    return 0;

+

+errout:

+    for(unsigned i = 0; i < alloc_count; i++)

+    {

+        PROTECT_rte_pktmbuf_free(mbufs[i]);

+        mbufs[i] = NULL;

+    }

+

+    return -1;

+#endif

+}

+

+static inline char * PROTECT_rte_pktmbuf_append(struct rte_mbuf *m, uint16_t len)

+{

+    PROTECT_rte_mbuf_unpoison_meta(m);

+    char * ret = rte_pktmbuf_append(m, len);

+    PROTECT_rte_mbuf_unpoison_data(m);

+    return ret;

+}

+

+static inline char * PROTECT_rte_pktmbuf_prepend(struct rte_mbuf *m, uint16_t len)

+{

+    PROTECT_rte_mbuf_unpoison_meta(m);

+    char * ret = rte_pktmbuf_prepend(m, len);

+    PROTECT_rte_mbuf_unpoison_data(m);

+    return ret;

+}

+

+static inline char * PROTECT_rte_pktmbuf_adj(struct rte_mbuf *m, uint16_t len)

+{

+    PROTECT_rte_mbuf_unpoison_meta(m);

+    char * ret = rte_pktmbuf_adj(m, len);

+    PROTECT_rte_mbuf_unpoison_data(m);

+    return ret;

+}

+

+static inline int PROTECT_rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len)

+{

+    PROTECT_rte_mbuf_unpoison_meta(m);

+    int ret = rte_pktmbuf_trim(m, len);

+    return ret;

+}

+

+static inline uint16_t PROTECT_rte_pktmbuf_data_len(struct rte_mbuf *m)

+{

+    PROTECT_rte_mbuf_unpoison_meta(m);

+    uint16_t ret = rte_pktmbuf_data_len(m);

+    return ret;

+}

+

+static inline uint32_t PROTECT_rte_pktmbuf_pkt_len(struct rte_mbuf *m)

+{

+    PROTECT_rte_mbuf_unpoison_meta(m);

+    uint32_t ret = rte_pktmbuf_pkt_len(m);

+    return ret;

+}

+

+static inline uint16_t PROTECT_rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)

+{

+    PROTECT_rte_mbuf_unpoison_meta(m);

+    uint16_t ret = rte_mbuf_refcnt_update(m, value);

+    return ret;

+}

+

+static inline uint16_t PROTECT_rte_mbuf_refcnt_read(const struct rte_mbuf *m)

+{

+    PROTECT_rte_mbuf_unpoison_meta((struct rte_mbuf *)m);

+    uint16_t ret = rte_mbuf_refcnt_read(m);

+    return ret;

+}
\ No newline at end of file
diff --git a/infra/include/vnode.h b/infra/include/vnode.h
index 9734117..714c774 100644
--- a/infra/include/vnode.h
+++ b/infra/include/vnode.h
@@ -36,6 +36,12 @@ int vnode_##_type##_delete_prod(struct vnode_prod * prod);
 
 #define __DECLARE_COMMON_VNODE_DELETE_CONS(_type)															\
 int vnode_##_type##_delete_cons(struct vnode_cons * cons);
+
+#define __DECLARE_COMMON_VNODE_UNPOISON_PROD(_type)															\
+void vnode_##_type##_unpoison_prod(struct vnode_prod * prod);
+
+#define __DECLARE_COMMON_VNODE_UNPOISON_CONS(_type)															\
+void vnode_##_type##_unpoison_cons(struct vnode_cons * cons);
 	
 struct vnode_cons_stat
 {
@@ -81,6 +87,8 @@ int vnode_mirror_delete(struct vnode * vnode);
 
 void vnode_mirror_flush(struct vnode_prod * prod, unsigned int prodq);
 
+void vnode_mirror_common_unpoison(struct vnode * vnode);
+
 __DECLARE_COMMON_VNODE_CREATE_PROD(mirror)
 __DECLARE_COMMON_VNODE_CREATE_CONS(mirror)
 __DECLARE_COMMON_VNODE_DELETE_PROD(mirror)
@@ -91,6 +99,8 @@ __DECLARE_COMMON_VNODE_PROD_STAT_GET(mirror)
 __DECLARE_COMMON_VNODE_CONS_STAT_GET(mirror)
 __DECLARE_COMMON_VNODE_PROD_ATTACH(mirror)
 __DECLARE_COMMON_VNODE_CONS_ATTACH(mirror)
+__DECLARE_COMMON_VNODE_UNPOISON_PROD(mirror)
+__DECLARE_COMMON_VNODE_UNPOISON_CONS(mirror)
 
 #ifdef __cplusplus
 }
diff --git a/infra/src/vnode_common.c b/infra/src/vnode_common.c
index 7cf5c80..bef6748 100644
--- a/infra/src/vnode_common.c
+++ b/infra/src/vnode_common.c
@@ -26,6 +26,7 @@
 #include <common.h>
 #include <rte_mbuf.h>
 
+#include <protect.h>
 #include "vnode_common.h"
 
 /* What is a tunnel ?
@@ -104,6 +105,38 @@ static int tunnel_block_delete(struct tunnel_block * block)
 	return 0;
 }
 
+static void tunnel_unpoison(struct tunnel_desc * desc)
+{
+	MR_ASAN_UNPOISON_MEMORY_REGION(desc, sizeof(struct tunnel_desc));
+	MR_ASAN_UNPOISON_MEMORY_REGION(desc->en_buffer, sizeof(void *) * desc->sz_en_buffer);
+
+	MR_ASAN_UNPOISON_MEMORY_REGION(desc->tunnel_object, sizeof(struct rte_ring));
+	ssize_t sz_tunnel_object = rte_ring_get_memsize(rte_ring_get_size(desc->tunnel_object));
+	assert(sz_tunnel_object >= 0);
+	MR_ASAN_UNPOISON_MEMORY_REGION(desc->tunnel_object, (size_t)sz_tunnel_object);
+}
+
+static void tunnel_block_unpoison(struct tunnel_block * block)
+{
+	/* Tunnel-Block Header */
+	MR_ASAN_UNPOISON_MEMORY_REGION(block, sizeof(struct tunnel_block));
+
+	/* Tunnel-Block Body */
+	void * __tunnel_block_body = (void *)block + sizeof(struct tunnel_block);
+	size_t __tunnel_block_body_size = sizeof(struct tunnel_desc *) * (block->nr_prodq * block->nr_consq);
+	MR_ASAN_UNPOISON_MEMORY_REGION(__tunnel_block_body, __tunnel_block_body_size);
+
+	/* Tunnel Objects */
+	for (unsigned int prodq_id = 0; prodq_id < block->nr_prodq; prodq_id++)
+	{
+		for (unsigned int consq_id = 0; consq_id < block->nr_consq; consq_id++)
+		{
+			struct tunnel_desc * tunnel_desc = *tunnel_block_locate(block, prodq_id, consq_id);
+			tunnel_unpoison(tunnel_desc);
+		}
+	}
+}
+
 /* Alloc a block of tunnels, and init all the tunnels */
 static struct tunnel_block * tunnel_block_new(const char * symbol, 
 	struct vnode_prod * prod, struct vnode_cons * cons, unsigned int tunnel_size, unsigned int tun_sz_buffer)
@@ -566,4 +599,23 @@ int __vnode_common_delete(struct vnode * vnode)
 
 	if (vnode != NULL) rte_free(vnode);
 	return 0;
+}
+
+void __vnode_common_unpoison(struct vnode * vnode)
+{
+	MR_ASAN_UNPOISON_MEMORY_REGION(vnode, sizeof(struct vnode));
+}
+
+void __vnode_common_unpoison_prod(struct vnode_prod * prod)
+{
+	MR_ASAN_UNPOISON_MEMORY_REGION(prod, sizeof(struct vnode_prod));
+	__vnode_common_unpoison(prod->vnode);
+	for(unsigned int i = 0; i < prod->nr_block; i++) tunnel_block_unpoison(prod->block_list[i]);
+}
+
+void __vnode_common_unpoison_cons(struct vnode_cons * cons)
+{
+	MR_ASAN_UNPOISON_MEMORY_REGION(cons, sizeof(struct vnode_cons));
+	__vnode_common_unpoison(cons->vnode);
+	for(unsigned int i = 0; i < cons->nr_block; i++) tunnel_block_unpoison(cons->block_list[i]);
 }
\ No newline at end of file
diff --git a/infra/src/vnode_common.h b/infra/src/vnode_common.h
index 207ce59..a63029d 100644
--- a/infra/src/vnode_common.h
+++ b/infra/src/vnode_common.h
@@ -146,6 +146,10 @@ int __vnode_common_delete_prod(struct vnode_prod * prod);
 int __vnode_common_delete_cons(struct vnode_cons * cons);
 int __vnode_common_delete(struct vnode * vnode);
 
+void __vnode_common_unpoison(struct vnode * vnode);
+void __vnode_common_unpoison_prod(struct vnode_prod * prod);
+void __vnode_common_unpoison_cons(struct vnode_cons * cons);
+
 #define __USE_COMMON_VNODE_CREATE_PROD(_type)																\
 struct vnode_prod * vnode_##_type##_create_prod(struct vnode * vnode, const char * symbol, int nr_prodq)	\
 {	return __vnode_common_create_prod(vnode,symbol,nr_prodq); }
@@ -186,3 +190,11 @@ int vnode_##_type##_delete_prod(struct vnode_prod * prod)													\
 int vnode_##_type##_delete_cons(struct vnode_cons * cons)												    \
 {	return __vnode_common_delete_cons(cons); }																
 
+#define __USE_COMMON_VNODE_UNPOISON_PROD(_type)																\
+void vnode_##_type##_unpoison_prod(struct vnode_prod * prod)												\
+{	return __vnode_common_unpoison_prod(prod); }
+
+#define __USE_COMMON_VNODE_UNPOISON_CONS(_type)																\
+void vnode_##_type##_unpoison_cons(struct vnode_cons * cons)											    \
+{	return __vnode_common_unpoison_cons(cons); }
+
diff --git a/infra/src/vnode_mirror.c b/infra/src/vnode_mirror.c
index a1a0deb..30ad935 100644
--- a/infra/src/vnode_mirror.c
+++ b/infra/src/vnode_mirror.c
@@ -11,6 +11,7 @@
 #include <rte_mbuf.h>
 #include <rte_spinlock.h>
 #include <rte_version.h>
+#include <protect.h>
 
 #include "vnode_common.h"
 
@@ -25,27 +26,27 @@ static inline void dist_tunnel_flush(struct vnode_prod * prod, struct vnode_cons
     int n_to_send = desc->sz_en_buffer_used;
     if (n_to_send == 0) return;
 
+	size_t n_send_len = 0;
+	for (unsigned int k = 0; k < n_to_send; k++)
+	{
+		n_send_len += PROTECT_rte_pktmbuf_data_len(desc->en_buffer[k]);
+	}
+
 #if RTE_VERSION >= RTE_VERSION_NUM(17,5,0,0)
 	int n_send = rte_ring_sp_enqueue_burst(desc->tunnel_object, (void **)desc->en_buffer, n_to_send, NULL);
 #else
     int n_send = rte_ring_sp_enqueue_burst(desc->tunnel_object, (void **)desc->en_buffer, n_to_send);
 #endif
 
-	size_t n_send_len = 0;
-	for (unsigned int k = 0; k < n_to_send; k++)
-	{
-		n_send_len += rte_pktmbuf_pkt_len(desc->en_buffer[k]);
-	}
-
 	/* 没有丢包 */
 	if (likely(n_send == n_to_send)) goto out;
 
     // release all the objects which not send successfully.
-    for (unsigned int k = n_send; k < n_to_send; k++)
+    for (int k = n_send; k < n_to_send; k++)
     {
         struct rte_mbuf * object_to_be_free = desc->en_buffer[k];
-		n_send_len -= rte_pktmbuf_pkt_len(object_to_be_free);
-        rte_pktmbuf_free(object_to_be_free);
+		n_send_len -= PROTECT_rte_pktmbuf_data_len(object_to_be_free);
+		PROTECT_rte_pktmbuf_free(object_to_be_free);
     }
 
 out:
@@ -60,7 +61,7 @@ out:
 	VNODE_STAT_UPDATE(cons, consq, deliver, n_send);
     VNODE_STAT_UPDATE(cons, consq, missed, n_to_send - n_send);
 	VNODE_STAT_UPDATE(cons, consq, total_len, n_send_len);
-	
+
     // 清空缓冲区
     desc->sz_en_buffer_used = 0;
     return;
@@ -76,6 +77,8 @@ static inline void dist_tunnel_enqueue(struct vnode_prod * prod, struct vnode_co
     // append the object at the tail of enqueue buffer.
 	unsigned int pos;
 	pos = desc->sz_en_buffer_used;
+
+	assert(pos < desc->sz_en_buffer);
 	desc->en_buffer[pos++] = obj;
 
 	// the enqueue buffer is not full, return
@@ -116,7 +119,7 @@ static inline void dist_tunnel_enqueue_without_buffer(struct tunnel_desc * desc,
 	int ret = rte_ring_sp_enqueue_burst(desc->tunnel_object, &obj, 1);
 #endif
 
-	if (unlikely(ret != 1)) rte_pktmbuf_free(obj);
+	if (unlikely(ret != 1)) PROTECT_rte_pktmbuf_free(obj);
 	return;
 }
 
@@ -124,7 +127,7 @@ static inline void dist_tunnel_enqueue_without_buffer(struct tunnel_desc * desc,
 // only use by prod.
 // TODO: rewrite in SSE/SSE2/AVX/AVX2 intrinsics
 
-static inline void dist_tunnel_block_enqueue_with_hash(struct tunnel_block * block, 
+static inline void dist_tunnel_block_enqueue_with_hash(struct tunnel_block * block,
 	int prodq, struct rte_mbuf * obj[], uint32_t hash[], int nr_obj)
 {
 	struct tunnel_desc * tunnel;
@@ -181,7 +184,7 @@ static inline void dist_tunnel_block_enqueue_with_hash(struct tunnel_block * blo
 // Tunnel Block Dequeue, dequeue from block, only used by cons.
 // TODO: rewrite in SSE/SSE2/AVX/AVX2 intrinsics
 
-static inline int dist_tunnel_block_dequeue(struct tunnel_block * block, 
+static inline int dist_tunnel_block_dequeue(struct tunnel_block * block,
 	int consq, struct rte_mbuf * obj[], int nr_max_obj)
 {
 	unsigned int nr_obj = 0, nr_obj_recv = 0;
@@ -224,7 +227,7 @@ static inline int __mirror_clone_objects(struct rte_mempool * clone_pool,
 {
     for (int i = 0; i < nr_source_objects; i++)
     {
-        rte_mbuf_refcnt_update(source_objects[i], 1);
+		PROTECT_rte_mbuf_refcnt_update(source_objects[i], 1);
     }
 
     rte_memcpy(cloned_objects, source_objects, sizeof(struct rte_mbuf *) * nr_source_objects);
@@ -233,8 +236,8 @@ static inline int __mirror_clone_objects(struct rte_mempool * clone_pool,
 
 #else
 
-static inline int __mirror_clone_objects(struct rte_mempool * clone_pool, 
-	struct rte_mbuf * source_objects[], struct rte_mbuf * cloned_objects[], 
+static inline int __mirror_clone_objects(struct rte_mempool * clone_pool,
+	struct rte_mbuf * source_objects[], struct rte_mbuf * cloned_objects[],
 	unsigned int nr_source_objects)
 {
 	// 全部复制成功标志位，避免在循环体内判断，减少分支判断造成的流水线中断
@@ -245,13 +248,13 @@ static inline int __mirror_clone_objects(struct rte_mempool * clone_pool,
 		cloned_objects[i] = rte_pktmbuf_clone(source_objects[i], clone_pool);
 		tag_clone_objects = (tag_clone_objects && cloned_objects[i]);
 	}
-	
+
 	if (likely(tag_clone_objects)) return 0;
 
 	/* 克隆失败，释放所有已经克隆的报文 */
 	for(int i = 0; i < nr_source_objects; i++)
 	{
-		rte_pktmbuf_free(cloned_objects[i]);
+		PROTECT_rte_pktmbuf_free(cloned_objects[i]);
 	}
 
 	return -1;
@@ -259,7 +262,7 @@ static inline int __mirror_clone_objects(struct rte_mempool * clone_pool,
 
 #endif
 
-int vnode_mirror_enqueue_bulk(struct vnode_prod * prod, 
+int vnode_mirror_enqueue_bulk(struct vnode_prod * prod,
 	unsigned int prodq, struct rte_mbuf * objects[], uint32_t hash[], int nr_objects)
 {
 	int ret = 0;
@@ -291,7 +294,7 @@ int vnode_mirror_enqueue_bulk(struct vnode_prod * prod,
             VNODE_STAT_UPDATE(prod, prodq, cloned_fail, nr_objects);
             continue;
         }
-        
+
 		dist_tunnel_block_enqueue_with_hash(block, prodq,
 			cloned_objects, hash, nr_objects);
 	}
@@ -301,9 +304,9 @@ int vnode_mirror_enqueue_bulk(struct vnode_prod * prod,
 		objects, hash, nr_objects);
 
     ret = 0; goto out;
-    
+
 failure:
-    for (int i = 0; i < nr_objects; i++) rte_pktmbuf_free(objects[i]);
+    for (int i = 0; i < nr_objects; i++) PROTECT_rte_pktmbuf_free(objects[i]);
 	VNODE_STAT_UPDATE(prod, prodq, on_line, nr_objects);
 	VNODE_STAT_UPDATE(prod, prodq, missed, nr_objects); 
     return 0;
@@ -376,6 +379,12 @@ int vnode_mirror_delete(struct vnode * vnode)
 	return __vnode_common_delete(vnode);
 }
 
+void vnode_mirror_common_unpoison(struct vnode * vnode)
+{
+	__vnode_common_unpoison(vnode);
+	MR_ASAN_UNPOISON_MEMORY_REGION(vnode->priv, sizeof(struct __vnode_priv_mirror));
+}
+
 __USE_COMMON_VNODE_CREATE_PROD(mirror)
 __USE_COMMON_VNODE_CREATE_CONS(mirror)
 __USE_COMMON_VNODE_DELETE_PROD(mirror)
@@ -385,4 +394,6 @@ __USE_COMMON_VNODE_PROD_LOOKUP(mirror)
 __USE_COMMON_VNODE_PROD_STAT_GET(mirror)
 __USE_COMMON_VNODE_CONS_STAT_GET(mirror)
 __USE_COMMON_VNODE_PROD_ATTACH(mirror)
-__USE_COMMON_VNODE_CONS_ATTACH(mirror)
\ No newline at end of file
+__USE_COMMON_VNODE_CONS_ATTACH(mirror)
+__USE_COMMON_VNODE_UNPOISON_PROD(mirror)
+__USE_COMMON_VNODE_UNPOISON_CONS(mirror)
\ No newline at end of file
diff --git a/service/src/core.c b/service/src/core.c
index 01e9733..65156ad 100644
--- a/service/src/core.c
+++ b/service/src/core.c
@@ -110,6 +110,8 @@ unsigned int g_logger_to_stdout = 1;
 unsigned int g_logger_level = LOG_DEBUG;
 unsigned int g_monit_interval = 1;
 unsigned int g_keep_running = 1;
+unsigned int g_in_protect_mode = 0;
+rte_spinlock_t g_in_protect_lock = RTE_SPINLOCK_INITIALIZER;
 
 static struct sc_main * g_sc_main = NULL;