[PACKET_IO]bugfix for marsio mode

11 files changed, 208 insertions, 153 deletions

diff --git a/sdk/include/utils.h b/sdk/include/utils.h
index 6dac461..450ab24 100644
--- a/sdk/include/utils.h
+++ b/sdk/include/utils.h
@@ -3,7 +3,7 @@
 #include <stdlib.h>	//calloc
 
 #define CALLOC(type, number) ((type *)calloc(sizeof(type), number))
-#define REALLOC(addr, number) ((void *)realloc(addr, number))
+
 #define	FREE(p)	{free(p);p=NULL;}
 
 #define	TRUE	1
diff --git a/src/main.cpp b/src/main.cpp
index 4f2774a..abdec48 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -60,7 +60,7 @@ packet_io_init(const char *instance_name, const enum packet_io_run_mode mode, co
 		return nullptr;
 	}
 
-	struct packet_io_device *ppio_dev = packet_io_open_device(ppio_inst, "eth1", wrk_thread_num, wrk_thread_num);
+	struct packet_io_device *ppio_dev = packet_io_device_open(ppio_inst, "eth1", wrk_thread_num, wrk_thread_num);
 	if (nullptr == ppio_dev) {
 		log_error(ST_ERR_PIO_DEVICE, "packet_io device open failed.");
 		
diff --git a/src/packet_io/marsio_mode/pio_marsio.cpp b/src/packet_io/marsio_mode/pio_marsio.cpp
index 0e2d4bc..92a54d5 100644
--- a/src/packet_io/marsio_mode/pio_marsio.cpp
+++ b/src/packet_io/marsio_mode/pio_marsio.cpp
@@ -18,9 +18,8 @@
 #include "../../sdk/include/utils.h"
 #include "../../sdk/include/util_errors.h"
 
-#define MARSIO_BURST_PKT_MAX		(256)
-
-static struct marsio_dll_function_entries shared_marsio_dll_func_entries;
+/* marsio dynamic link lib function entries */
+static struct marsio_dll_function_entries g_marsio_dll_func;
 
 static void fake_marsio_buff_set_rehash_index(marsio_buff_t *m, uint32_t hash)
 {
@@ -30,200 +29,205 @@ static void fake_marsio_buff_set_rehash_index(marsio_buff_t *m, uint32_t hash)
 static int pio_get_marsio_dll_function_entries(void) 
 {
 	
-    void *marsio_so_handle = dlopen(g_engine_instance.config.lib.libmarsio_path, RTLD_NOW | RTLD_LOCAL | RTLD_NODELETE);
+    void *marsio_so_handle = dlopen(g_engine_instance.config.lib.libmarsio_path, 
+									RTLD_NOW | RTLD_LOCAL | RTLD_NODELETE);
     if (nullptr == marsio_so_handle) {
-        printf("\033[1;31;40m[Error]dlopen '%s' failed, %s\033[0m\n", g_engine_instance.config.lib.libmarsio_path, dlerror());
+        printf("\033[1;31;40m[Error]dlopen '%s' failed, %s\033[0m\n", 
+				g_engine_instance.config.lib.libmarsio_path, dlerror());
 		return -1;
     }
 	
-    shared_marsio_dll_func_entries.ptr_marsio_create = 
-        (struct mr_instance *(*)(void))dlsym(marsio_so_handle, "marsio_create");
-    if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_create) {
+    g_marsio_dll_func.marsio_create = \
+		(struct mr_instance *(*)(void))dlsym(marsio_so_handle, "marsio_create");
+    if (nullptr == g_marsio_dll_func.marsio_create) {
         printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_create", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
     }
 
-    shared_marsio_dll_func_entries.ptr_marsio_init = 
-        (int (*)(struct mr_instance *, const char *))dlsym(marsio_so_handle, "marsio_init");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_init) {
+    g_marsio_dll_func.marsio_init = \
+		(int (*)(struct mr_instance *, const char *))dlsym(marsio_so_handle, "marsio_init");
+	if (nullptr == g_marsio_dll_func.marsio_init) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_init", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-    shared_marsio_dll_func_entries.ptr_marsio_destroy = 
-        (int (*)(struct mr_instance *))dlsym(marsio_so_handle, "marsio_destory");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_destroy) {
+    g_marsio_dll_func.marsio_destroy = \
+		(int (*)(struct mr_instance *))dlsym(marsio_so_handle, "marsio_destory");
+	if (nullptr == g_marsio_dll_func.marsio_destroy) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_destory", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-    shared_marsio_dll_func_entries.ptr_marsio_open_device = 
-        (struct mr_vdev *(*)(struct mr_instance *, const char *, unsigned int, unsigned int))dlsym(marsio_so_handle, "marsio_open_device");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_open_device) {
+    g_marsio_dll_func.marsio_open_device = \
+        (struct mr_vdev *(*)(struct mr_instance *, const char *, 
+							 unsigned int, unsigned int))dlsym(marsio_so_handle, "marsio_open_device");
+	if (nullptr == g_marsio_dll_func.marsio_open_device) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_open_device", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-    shared_marsio_dll_func_entries.ptr_marsio_close_device = 
-        (void (*)(struct mr_vdev *))dlsym(marsio_so_handle, "marsio_close_device");
-	if (nullptr ==shared_marsio_dll_func_entries.ptr_marsio_close_device) {
+    g_marsio_dll_func.marsio_close_device = \
+		(void (*)(struct mr_vdev *))dlsym(marsio_so_handle, "marsio_close_device");
+	if (nullptr ==g_marsio_dll_func.marsio_close_device) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_close_device", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-    shared_marsio_dll_func_entries.ptr_marsio_option_set = 
+    g_marsio_dll_func.marsio_option_set = \
         (int (*)(struct mr_instance *, marsio_opt_type_t, void *, size_t))dlsym(marsio_so_handle, "marsio_option_set");
-    if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_option_set) {
+    if (nullptr == g_marsio_dll_func.marsio_option_set) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_option_set", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-    shared_marsio_dll_func_entries.ptr_marsio_sendpath_create_by_vdev = 
+    g_marsio_dll_func.marsio_sendpath_create_by_vdev = \
         (struct mr_sendpath *(*)(struct mr_vdev *))dlsym(marsio_so_handle, "marsio_sendpath_create_by_vdev");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_sendpath_create_by_vdev) {
+	if (nullptr == g_marsio_dll_func.marsio_sendpath_create_by_vdev) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_sendpath_create_by_vdev", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-	shared_marsio_dll_func_entries.ptr_marsio_sendpath_destroy = 
+	g_marsio_dll_func.marsio_sendpath_destroy = \
         (void (*)(struct mr_sendpath *))dlsym(marsio_so_handle, "marsio_sendpath_destory");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_sendpath_destroy) {
+	if (nullptr == g_marsio_dll_func.marsio_sendpath_destroy) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_sendpath_destory", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 	
-	shared_marsio_dll_func_entries.ptr_marsio_thread_init = 
-        (int (*)(struct mr_instance *))dlsym(marsio_so_handle, "marsio_thread_init");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_thread_init) {
+	g_marsio_dll_func.marsio_thread_init = (int (*)(struct mr_instance *))dlsym(marsio_so_handle, "marsio_thread_init");
+	if (nullptr == g_marsio_dll_func.marsio_thread_init) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_thread_init", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-	shared_marsio_dll_func_entries.ptr_marsio_recv_burst = 
+	g_marsio_dll_func.marsio_recv_burst = \
         (int (*)(struct mr_vdev *, queue_id_t, marsio_buff_t **, int))dlsym(marsio_so_handle, "marsio_recv_burst");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_recv_burst) {
+	if (nullptr == g_marsio_dll_func.marsio_recv_burst) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_recv_burst", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-	shared_marsio_dll_func_entries.ptr_marsio_send_burst = 
+	g_marsio_dll_func.marsio_send_burst = \
         (int (*)(struct mr_sendpath *,  queue_id_t, marsio_buff_t **, int))dlsym(marsio_so_handle, "marsio_send_burst");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_send_burst) {
+	if (nullptr == g_marsio_dll_func.marsio_send_burst) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_send_burst", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-	shared_marsio_dll_func_entries.ptr_marsio_send_burst_with_options = 
-        (int (*)(struct mr_sendpath *, queue_id_t, marsio_buff_t **, int, uint16_t))dlsym(marsio_so_handle, "marsio_send_burst_with_options");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_send_burst_with_options) {
+	g_marsio_dll_func.marsio_send_burst_with_options = 
+        (int (*)(struct mr_sendpath *, queue_id_t, 
+				 marsio_buff_t **, int, uint16_t))dlsym(marsio_so_handle, "marsio_send_burst_with_options");
+	if (nullptr == g_marsio_dll_func.marsio_send_burst_with_options) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_send_burst_with_options", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-    shared_marsio_dll_func_entries.ptr_marsio_buff_malloc_global = 
-        (int (*)(struct mr_instance *, marsio_buff_t **, unsigned int, int, int))dlsym(marsio_so_handle, "marsio_buff_malloc_global");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_buff_malloc_global) {
+    g_marsio_dll_func.marsio_buff_malloc_global = \
+        (int (*)(struct mr_instance *, marsio_buff_t **, 
+				 unsigned int, int, int))dlsym(marsio_so_handle, "marsio_buff_malloc_global");
+	if (nullptr == g_marsio_dll_func.marsio_buff_malloc_global) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_buff_malloc_global", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-	shared_marsio_dll_func_entries.ptr_marsio_buff_free = 
-        (void (*)(struct mr_instance *, marsio_buff_t **,unsigned int, int, int))dlsym(marsio_so_handle, "marsio_buff_free");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_buff_free) {
+	g_marsio_dll_func.marsio_buff_free = \
+        (void (*)(struct mr_instance *, marsio_buff_t **,
+				  unsigned int, int, int))dlsym(marsio_so_handle, "marsio_buff_free");
+	if (nullptr == g_marsio_dll_func.marsio_buff_free) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_buff_free", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-	shared_marsio_dll_func_entries.ptr_marsio_buff_append = 
+	g_marsio_dll_func.marsio_buff_append = \
         (char * (*)(marsio_buff_t *, uint16_t))dlsym(marsio_so_handle, "marsio_buff_append");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_buff_append) {
+	if (nullptr == g_marsio_dll_func.marsio_buff_append) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_buff_append", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-	shared_marsio_dll_func_entries.ptr_marsio_buff_ctrlzone = 
+	g_marsio_dll_func.marsio_buff_ctrlzone = \
         (void * (*)(marsio_buff_t *, uint8_t))dlsym(marsio_so_handle, "marsio_buff_ctrlzone");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_buff_ctrlzone) {
+	if (nullptr == g_marsio_dll_func.marsio_buff_ctrlzone) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_buff_ctrlzone", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-	shared_marsio_dll_func_entries.ptr_marsio_buff_ctrlzone_set = 
-        (void (*)(marsio_buff_t *, uint8_t, void *, uint8_t))dlsym(marsio_so_handle, "marsio_buff_ctrlzone_set");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_buff_ctrlzone_set) {
+	g_marsio_dll_func.marsio_buff_ctrlzone_set = \
+        (void (*)(marsio_buff_t *, uint8_t, 
+				  void *, uint8_t))dlsym(marsio_so_handle, "marsio_buff_ctrlzone_set");
+	if (nullptr == g_marsio_dll_func.marsio_buff_ctrlzone_set) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_buff_ctrlzone_set", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-	shared_marsio_dll_func_entries.ptr_marsio_buff_set_rehash_index = 
+	g_marsio_dll_func.marsio_buff_set_rehash_index = \
         (void (*)(marsio_buff_t *, uint32_t))dlsym(marsio_so_handle, "marsio_buff_set_rehash_index");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_buff_set_rehash_index) {
+	if (nullptr == g_marsio_dll_func.marsio_buff_set_rehash_index) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_buff_set_rehash_index", 
 				g_engine_instance.config.lib.libmarsio_path);
-		shared_marsio_dll_func_entries.ptr_marsio_buff_set_rehash_index = fake_marsio_buff_set_rehash_index;
+		g_marsio_dll_func.marsio_buff_set_rehash_index = fake_marsio_buff_set_rehash_index;
 	}
 
-	shared_marsio_dll_func_entries.ptr_marsio_buff_mtod = 
-        (char * (*)(marsio_buff_t *))dlsym(marsio_so_handle, "marsio_buff_mtod");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_buff_mtod) {
+	g_marsio_dll_func.marsio_buff_mtod = (char * (*)(marsio_buff_t *))dlsym(marsio_so_handle, "marsio_buff_mtod");
+	if (nullptr == g_marsio_dll_func.marsio_buff_mtod) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_buff_mtod", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-	shared_marsio_dll_func_entries.ptr_marsio_buff_datalen = 
-        (uint32_t (*)(marsio_buff_t *))dlsym(marsio_so_handle, "marsio_buff_datalen");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_buff_datalen) {
+	g_marsio_dll_func.marsio_buff_datalen = (uint32_t (*)(marsio_buff_t *))dlsym(marsio_so_handle, "marsio_buff_datalen");
+	if (nullptr == g_marsio_dll_func.marsio_buff_datalen) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_buff_datalen", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-	shared_marsio_dll_func_entries.ptr_marsio_buff_buflen = 
-        (uint32_t (*)(marsio_buff_t *))dlsym(marsio_so_handle, "marsio_buff_buflen");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_buff_buflen) {
+	g_marsio_dll_func.marsio_buff_buflen = (uint32_t (*)(marsio_buff_t *))dlsym(marsio_so_handle, "marsio_buff_buflen");
+	if (nullptr == g_marsio_dll_func.marsio_buff_buflen) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_buff_buflen", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
-	shared_marsio_dll_func_entries.ptr_marsio_buff_clone_with_options = 
-        (marsio_buff_t * (*)(struct mr_instance *, marsio_buff_t *, int, int, uint16_t))dlsym(marsio_so_handle, "marsio_buff_clone_with_options");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_buff_clone_with_options) {
+	g_marsio_dll_func.marsio_buff_clone_with_options = \
+        (marsio_buff_t *(*)(struct mr_instance *, marsio_buff_t *, 
+							int, int, uint16_t))dlsym(marsio_so_handle, "marsio_buff_clone_with_options");
+	if (nullptr == g_marsio_dll_func.marsio_buff_clone_with_options) {
 		printf("\033[1;31;40m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_buff_clone_with_options", 
 				g_engine_instance.config.lib.libmarsio_path);
 		return -1;
 	}
 
 	/* note: if not include marsio_send_burst_flush(), no errors. include just for backward compatibility */
-	shared_marsio_dll_func_entries.ptr_marsio_send_burst_flush = 
+	g_marsio_dll_func.marsio_send_burst_flush = \
         (void (*)(struct mr_sendpath *, queue_id_t))dlsym(marsio_so_handle, "marsio_send_burst_flush");
-	if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_send_burst_flush) {
+	if (nullptr == g_marsio_dll_func.marsio_send_burst_flush) {
 		printf("\033[33m[Warning]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_send_burst_flush", 
 				g_engine_instance.config.lib.libmarsio_path);
 	}
 
     /* for vlan flipping */
-    shared_marsio_dll_func_entries.ptr_marsio_buff_get_metadata = 
-        (int (*)(marsio_buff_t *, enum mr_buff_metadata_type, void *, unsigned int))dlsym(marsio_so_handle, "marsio_buff_get_metadata");
-    if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_buff_get_metadata) {
+    g_marsio_dll_func.marsio_buff_get_metadata = \
+        (int (*)(marsio_buff_t *, enum mr_buff_metadata_type, 
+				 void *, unsigned int))dlsym(marsio_so_handle, "marsio_buff_get_metadata");
+    if (nullptr == g_marsio_dll_func.marsio_buff_get_metadata) {
         printf("\033[33m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_buff_get_metadata", 
 				g_engine_instance.config.lib.libmarsio_path);
         /* 
@@ -232,9 +236,10 @@ static int pio_get_marsio_dll_function_entries(void)
 		*/
     }
 
-    shared_marsio_dll_func_entries.ptr_marsio_buff_set_metadata = 
-        (int (*)(marsio_buff_t *, enum mr_buff_metadata_type, void *, unsigned int))dlsym(marsio_so_handle, "marsio_buff_set_metadata");
-    if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_buff_get_metadata) {
+    g_marsio_dll_func.marsio_buff_set_metadata = \
+        (int (*)(marsio_buff_t *, enum mr_buff_metadata_type, 
+				 void *, unsigned int))dlsym(marsio_so_handle, "marsio_buff_set_metadata");
+    if (nullptr == g_marsio_dll_func.marsio_buff_get_metadata) {
         printf("\033[33m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_buff_set_metadata", 
 				g_engine_instance.config.lib.libmarsio_path);
         /* 
@@ -243,9 +248,9 @@ static int pio_get_marsio_dll_function_entries(void)
 		*/
     }
 
-    shared_marsio_dll_func_entries.ptr_marsio_buff_unset_metadata = 
+    g_marsio_dll_func.marsio_buff_unset_metadata = \
         (int (*)(marsio_buff_t *, enum mr_buff_metadata_type))dlsym(marsio_so_handle, "marsio_buff_unset_metadata");
-    if (nullptr == shared_marsio_dll_func_entries.ptr_marsio_buff_unset_metadata) {
+    if (nullptr == g_marsio_dll_func.marsio_buff_unset_metadata) {
         printf("\033[33m[Error]dlsym function '%s' from '%s' failed!\033[0m\n", "marsio_buff_unset_metadata", 
 				g_engine_instance.config.lib.libmarsio_path);
         /* 
@@ -260,13 +265,23 @@ static int pio_get_marsio_dll_function_entries(void)
 int pio_marsio_device_open(struct packet_io_device *pdev, const char *dev_name, uint32_t nr_rxq, uint32_t nr_txq) 
 {
 	struct mr_instance *mr_inst_handle = pdev->ppio_inst->entity.marsio_inst->mr_inst_handle;
+
+	/* marsio_open_device() return marsio device handle*/
 	pdev->entity.marsio_dev->mr_dev_handle = \
-			shared_marsio_dll_func_entries.ptr_marsio_open_device(mr_inst_handle, dev_name, nr_rxq, nr_txq);
+			g_marsio_dll_func.marsio_open_device(mr_inst_handle, dev_name, nr_rxq, nr_txq);
 	if (nullptr == pdev->entity.marsio_dev->mr_dev_handle) {
 		fprintf(stderr,"marsio_open_device:%s error\n", dev_name);
 		return -1;
 	}
 	
+	/* ptr_marsio_sendpath_create_by_vdev() return marsio sendpath handle */
+	pdev->entity.marsio_dev->mr_sendpath_handle = \
+			g_marsio_dll_func.marsio_sendpath_create_by_vdev(pdev->entity.marsio_dev->mr_dev_handle);
+	if (nullptr == pdev->entity.marsio_dev->mr_sendpath_handle) {
+		log_error(ST_ERR_PIO_DEVICE, "device:%s marsio_sendpath_create_by_vdev failed!", pdev->dev_name);
+		return -1;
+	}
+
     return 0;
 }
 
@@ -277,29 +292,22 @@ int pio_marsio_device_close(const struct packet_io_device *pdev)
 		return -1;
 	}
 
-	shared_marsio_dll_func_entries.ptr_marsio_close_device(pdev->entity.marsio_dev->mr_dev_handle);
+	g_marsio_dll_func.marsio_close_device(pdev->entity.marsio_dev->mr_dev_handle);
+	g_marsio_dll_func.marsio_sendpath_destroy(pdev->entity.marsio_dev->mr_sendpath_handle);
 
     return 0;
 }
 
 int pio_marsio_device_receive(struct packet_io_device *pdev, uint32_t rxq_id, struct packet *p[], int nr_p) 
 {
-	return shared_marsio_dll_func_entries.ptr_marsio_recv_burst(pdev->entity.marsio_dev->mr_dev_handle, 
-																rxq_id, (marsio_buff_t **)p, nr_p);
+	struct mr_vdev *mr_dev_handle = pdev->entity.marsio_dev->mr_dev_handle;
+	return g_marsio_dll_func.marsio_recv_burst(mr_dev_handle, rxq_id, (marsio_buff_t **)p, nr_p);
 }
 
 int pio_marsio_device_send(struct packet_io_device *pdev, uint32_t txq_id, struct packet *p[], int nr_p) 
 {
-	struct mr_sendpath *sendpath = \
-			shared_marsio_dll_func_entries.ptr_marsio_sendpath_create_by_vdev(pdev->entity.marsio_dev->mr_dev_handle);
-	if (nullptr == sendpath) {
-		log_error(ST_ERR_PIO_DEVICE, "device:%s marsio_sendpath_create_by_vdev failed!", pdev->dev_name);
-		return -1;
-	}
-
-	shared_marsio_dll_func_entries.ptr_marsio_send_burst(sendpath, txq_id, (marsio_buff_t **)p, nr_p);
-	
-    return 0;
+	struct mr_sendpath *sendpath_handle = pdev->entity.marsio_dev->mr_sendpath_handle;
+	return g_marsio_dll_func.marsio_send_burst(sendpath_handle, txq_id, (marsio_buff_t **)p, nr_p);
 }
 
 static int marsio_instance_init(struct packet_io_instance *pinst, int wrk_thread_num) 
@@ -311,16 +319,16 @@ static int marsio_instance_init(struct packet_io_instance *pinst, int wrk_thread
 		return -1;
     }
 
-	pinst->entity.marsio_inst->mr_inst_handle = shared_marsio_dll_func_entries.ptr_marsio_create();
+	pinst->entity.marsio_inst->mr_inst_handle = g_marsio_dll_func.marsio_create();
 	if (nullptr == pinst->entity.marsio_inst->mr_inst_handle) {
 		fprintf(stderr,"%s\n","marsio_create error!\n");
 		return -1;
 	}
 
 	/* TODO: MARSIO_OPT_THREAD_NUM */
-	ret = shared_marsio_dll_func_entries.ptr_marsio_option_set(pinst->entity.marsio_inst->mr_inst_handle, 
-														 	   MARSIO_OPT_THREAD_NUM, 
-														 	   &wrk_thread_num, sizeof(int));
+	ret = g_marsio_dll_func.marsio_option_set(pinst->entity.marsio_inst->mr_inst_handle, 
+											  MARSIO_OPT_THREAD_NUM, 
+											  &wrk_thread_num, sizeof(int));
 	if (ret < 0) {
 		fprintf(stderr,"%s\n","marsio_option_set MARSIO_OPT_THREAD_NUM error!\n");
 		return -1;
@@ -329,7 +337,7 @@ static int marsio_instance_init(struct packet_io_instance *pinst, int wrk_thread
 	/* TODO: MARSIO_OPT_THREAD_MASK_IN_CPUSET */
 
 	/* marsio_init */
-	ret = shared_marsio_dll_func_entries.ptr_marsio_init(pinst->entity.marsio_inst->mr_inst_handle, pinst->inst_name);
+	ret = g_marsio_dll_func.marsio_init(pinst->entity.marsio_inst->mr_inst_handle, pinst->inst_name);
 	if (ret < 0) {
 		fprintf(stderr,"%s\n","marsio_init error!\n");
 		return -1;
@@ -355,7 +363,12 @@ int pio_marsio_instance_create(struct packet_io_instance *pinst, int wrk_thread_
     return 0;
 }
 
-void pio_marsio_instance_destroy(void) 
+void pio_marsio_instance_destroy(struct packet_io_instance *pinst) 
 {
+	g_marsio_dll_func.marsio_destroy(pinst->entity.marsio_inst->mr_inst_handle);
 
+	for (uint32_t i = 0; i < pinst->dev_cnt; i++) {
+		pio_marsio_device_close(pinst->devices[i]);
+		FREE(pinst->devices[i]);
+	}
 }
\ No newline at end of file
diff --git a/src/packet_io/marsio_mode/pio_marsio.h b/src/packet_io/marsio_mode/pio_marsio.h
index 360d485..dc5e702 100644
--- a/src/packet_io/marsio_mode/pio_marsio.h
+++ b/src/packet_io/marsio_mode/pio_marsio.h
@@ -20,94 +20,102 @@
 */
 struct marsio_dll_function_entries {
     /* marsio_create() */
-    struct mr_instance *(*ptr_marsio_create)(void);
+    struct mr_instance *(*marsio_create)(void);
 
     /* marsio_init() */
-    int (*ptr_marsio_init)(struct mr_instance *instance, const char *appsym);
+    int (*marsio_init)(struct mr_instance *instance, const char *appsym);
 
     /* marsio_destroy() */
-    int (*ptr_marsio_destroy)(struct mr_instance *instance);
+    int (*marsio_destroy)(struct mr_instance *instance);
 
     /* marsio_open_device() */
-    struct mr_vdev *(*ptr_marsio_open_device)(struct mr_instance *instance, const char *devsym, 
+    struct mr_vdev *(*marsio_open_device)(struct mr_instance *instance, const char *devsym, 
                                               unsigned int nr_rxstream, unsigned int nr_txstream);
     
     /* marsio_close_device() */
-    void (*ptr_marsio_close_device)(struct mr_vdev *pdev);
+    void (*marsio_close_device)(struct mr_vdev *pdev);
 
     /* marsio_option_set() */
-    int (*ptr_marsio_option_set)(struct mr_instance *instance, marsio_opt_type_t opt_type, void *opt, size_t sz_opt);
+    int (*marsio_option_set)(struct mr_instance *instance, marsio_opt_type_t opt_type, void *opt, size_t sz_opt);
 
     /* marsio_sendpath_create_by_vdev() */
-    struct mr_sendpath *(*ptr_marsio_sendpath_create_by_vdev)(struct mr_vdev *pdev);
+    struct mr_sendpath *(*marsio_sendpath_create_by_vdev)(struct mr_vdev *pdev);
 
     /* marsio_sendpath_destroy() */
-    void (*ptr_marsio_sendpath_destroy)(struct mr_sendpath *sendpath);
+    void (*marsio_sendpath_destroy)(struct mr_sendpath *sendpath);
 
     /* marsio_thread_init() */
-    int (*ptr_marsio_thread_init)(struct mr_instance *instance);
+    int (*marsio_thread_init)(struct mr_instance *instance);
 
     /* marsio_recv_burst() */
-    int (*ptr_marsio_recv_burst)(struct mr_vdev *pdev, queue_id_t qid, marsio_buff_t *mbufs[], int nr_mbufs);
+    int (*marsio_recv_burst)(struct mr_vdev *pdev, queue_id_t qid, marsio_buff_t *mbufs[], int nr_mbufs);
 
     /* marsio_send_burst() */
-    int (*ptr_marsio_send_burst)(struct mr_sendpath *sendpath,  queue_id_t qid, marsio_buff_t *mbufs[], int nr_mbufs);
+    int (*marsio_send_burst)(struct mr_sendpath *sendpath,  queue_id_t qid, marsio_buff_t *mbufs[], int nr_mbufs);
 
     /* marsio_send_burst_with_options() */
-    int (*ptr_marsio_send_burst_with_options)(struct mr_sendpath *sendpath, queue_id_t sid, 
+    int (*marsio_send_burst_with_options)(struct mr_sendpath *sendpath, queue_id_t sid, 
                                               marsio_buff_t *mbufs[], int nr_mbufs, uint16_t options);
 
     /* marsio_buff_malloc_global() */
-    int (*ptr_marsio_buff_malloc_global)(struct mr_instance *instance, marsio_buff_t *marsio_buff[], unsigned int nr_mbufs, 
+    int (*marsio_buff_malloc_global)(struct mr_instance *instance, marsio_buff_t *marsio_buff[], unsigned int nr_mbufs, 
                                          int socket_id, int thread_id);
 
     /* marsio_buff_free() */
-    void (*ptr_marsio_buff_free)(struct mr_instance *, marsio_buff_t **,unsigned int, int socket_id, int thread_id);
+    void (*marsio_buff_free)(struct mr_instance *, marsio_buff_t **,unsigned int, int socket_id, int thread_id);
 
     /* marsio_buff_append() */
-    char *(*ptr_marsio_buff_append)(marsio_buff_t *m, uint16_t len);
+    char *(*marsio_buff_append)(marsio_buff_t *m, uint16_t len);
 
     /* marsio_buff_ctrlzone() */
-    void *(*ptr_marsio_buff_ctrlzone)(marsio_buff_t *m, uint8_t id);
+    void *(*marsio_buff_ctrlzone)(marsio_buff_t *m, uint8_t id);
 
     /* marsio_buff_ctrlzone_set() */
-    void (*ptr_marsio_buff_ctrlzone_set)(marsio_buff_t *m, uint8_t id, void* ptr_data, uint8_t size);
+    void (*marsio_buff_ctrlzone_set)(marsio_buff_t *m, uint8_t id, void* ptr_data, uint8_t size);
 
     /* marsio_buff_set_rehash_index() */
-    void (*ptr_marsio_buff_set_rehash_index)(marsio_buff_t *m, uint32_t hash);
+    void (*marsio_buff_set_rehash_index)(marsio_buff_t *m, uint32_t hash);
 
     /* marsio_buff_mtod() */
-    char *(*ptr_marsio_buff_mtod)(marsio_buff_t *m);
+    char *(*marsio_buff_mtod)(marsio_buff_t *m);
 
     /* marsio_buff_datalen() */
-    uint32_t (*ptr_marsio_buff_datalen)(marsio_buff_t *m);
+    uint32_t (*marsio_buff_datalen)(marsio_buff_t *m);
 
     /* marsio_buff_buflen() */
-    uint32_t (*ptr_marsio_buff_buflen)(marsio_buff_t *m);
+    uint32_t (*marsio_buff_buflen)(marsio_buff_t *m);
 
     /* marsio_buff_clone_with_options() */
-    marsio_buff_t *(*ptr_marsio_buff_clone_with_options)(struct mr_instance *instance, marsio_buff_t *md, int socket_id, 
+    marsio_buff_t *(*marsio_buff_clone_with_options)(struct mr_instance *instance, marsio_buff_t *md, int socket_id, 
                                                          int thread_id, uint16_t options);
 
     /* marsio_send_burst_flush() */
-    void (*ptr_marsio_send_burst_flush)(struct mr_sendpath *sendpath, queue_id_t sid);
+    void (*marsio_send_burst_flush)(struct mr_sendpath *sendpath, queue_id_t sid);
 
     /* marsio_buff_get_metadata() */
-    int (*ptr_marsio_buff_get_metadata)(marsio_buff_t *m, enum mr_buff_metadata_type type, void *data, unsigned int sz_data);
+    int (*marsio_buff_get_metadata)(marsio_buff_t *m, enum mr_buff_metadata_type type, void *data, unsigned int sz_data);
 
     /* marsio_buff_set_metadata() */
-    int (*ptr_marsio_buff_set_metadata)(marsio_buff_t *m, enum mr_buff_metadata_type type, void *data, unsigned int sz_data);
+    int (*marsio_buff_set_metadata)(marsio_buff_t *m, enum mr_buff_metadata_type type, void *data, unsigned int sz_data);
 
     /* marsio_bufrf_unset_metadata() */
-    int (*ptr_marsio_buff_unset_metadata)(marsio_buff_t *m, enum mr_buff_metadata_type type);
+    int (*marsio_buff_unset_metadata)(marsio_buff_t *m, enum mr_buff_metadata_type type);
 };
 
 struct pio_marsio_instance {
     struct mr_instance *mr_inst_handle;
 };
 
+/*  
+ * struct pio_marsio_device - marsio device abstract 
+ * @mr_dev_handle:      marsio device handle 
+ *                      if marsio device receive packets, use mr_dev_handle
+ * @mr_sendpath_handle: marsio sendpath handle
+ *                      if marsio device send packets, both mr_dev_handle and sendpath are required
+**/
 struct pio_marsio_device {
     struct mr_vdev *mr_dev_handle;
+    struct mr_sendpath * mr_sendpath_handle;
 };
 
 struct pio_marsio_device_context {
@@ -135,4 +143,4 @@ int pio_marsio_device_send(struct packet_io_device *pdev, uint32_t txq_id, struc
 
 int pio_marsio_instance_create(struct packet_io_instance *pinst, int wrk_thread_num);
 
-void pio_marsio_instance_destroy(void);
\ No newline at end of file
+void pio_marsio_instance_destroy(struct packet_io_instance *pinst);
\ No newline at end of file
diff --git a/src/packet_io/packet_io.cpp b/src/packet_io/packet_io.cpp
index 29763e6..d2cb687 100644
--- a/src/packet_io/packet_io.cpp
+++ b/src/packet_io/packet_io.cpp
@@ -41,7 +41,6 @@ struct pio_device_operations pio_device_ops_array[PACKET_IO_RUN_MODE_MAX] =
 
 struct pio_instance_operations pio_instance_ops_array[PACKET_IO_RUN_MODE_MAX] = 
 {
-
 	{
 		.create = pio_pcap_file_instance_create,
 		.destroy = pio_pcap_file_instance_destroy,
@@ -82,15 +81,16 @@ packet_io_instance_create(const char *instance_name, const enum packet_io_run_mo
 }
 
 void packet_io_instance_destroy(struct packet_io_instance *pinst) {
-	if (nullptr != pinst) {
-		if (nullptr != pinst->devices) {
-		
-		}
+	if (nullptr == pinst) {
+		return;
 	}
+
+	pinst->inst_ops->destroy(pinst);
+	FREE(pinst);
 }
 
 struct packet_io_device *
-packet_io_open_device(struct packet_io_instance *pinst, const char *dev_name, uint32_t nr_rxq, uint32_t nr_txq)
+packet_io_device_open(struct packet_io_instance *pinst, const char *dev_name, uint32_t nr_rxq, uint32_t nr_txq)
 {
 	struct packet_io_device *ppio_dev = CALLOC(struct packet_io_device, 1);
 	if (nullptr == ppio_dev) {
@@ -121,16 +121,23 @@ packet_io_open_device(struct packet_io_instance *pinst, const char *dev_name, ui
 	return ppio_dev;
 }
 
-void packet_io_close_device(struct packet_io_device *pdev)
+void packet_io_device_close(struct packet_io_device *pdev)
 {
-	if (nullptr == pdev || nullptr == pdev->dev_ops) {
+	if (nullptr == pdev) {
 		return;
 	}
 
-	int ret = pdev->dev_ops->close(nullptr);
+	if (nullptr == pdev->dev_ops) {
+		FREE(pdev);
+		return;
+	}
+
+	int ret = pdev->dev_ops->close(pdev);
 	if (ret < 0) {
 		log_error(ST_ERR_PIO_DEVICE, "packet_io device close failed.");
 	}
+
+	FREE(pdev);
 }
 
 int packet_io_device_rx(struct packet_io_device *pdev, uint32_t rxq_id, struct packet *p[], int nr_p)
diff --git a/src/packet_io/packet_io.h b/src/packet_io/packet_io.h
index ec8d10c..7514907 100644
--- a/src/packet_io/packet_io.h
+++ b/src/packet_io/packet_io.h
@@ -20,14 +20,15 @@
 #include "./marsio_mode/pio_marsio.h"
 
 /*
-	note: 
-		1. packet_io_XXX function is supported by packet_io.h
-		2. pio_XXX function is supported by pio_pcap_live.h/pio_pcap_file.h/pio_marsio.h
-*/
+ * note: 
+	1. packet_io_XXX function is supported by packet_io.h
+	2. pio_XXX function is supported by pio_pcap_live.h/pio_pcap_file.h/pio_marsio.h
+**/
+
 struct pio_instance_operations {
-	int (*create)(struct packet_io_instance *instance, int wrk_thread_num);
+	int (*create)(struct packet_io_instance *pinst, int wrk_thread_num);
 
-	void (*destroy)(void);
+	void (*destroy)(struct packet_io_instance *pinst);
 };
 
 struct packet_io_instance {
@@ -85,22 +86,48 @@ struct packet_io_device {
 };
 
 /*
-	@brief create packet_io instance which will manage packet_io device
-
-	@param instance_name:     packet_io instance name
-	@param mode:              packet_io run mode
-	@param wrk_thread_num:    expected number of packet receiving threads, which will be created by packet_io
-*/
+ * @brief create packet_io instance which will manage packet_io device
+ *
+ * @param instance_name:     packet_io instance name
+ * @param mode:              packet_io run mode
+ * @param wrk_thread_num:    expected number of packet receiving threads, which will be created by packet_io
+**/
 struct packet_io_instance *
 packet_io_instance_create(const char *instance_name, const enum packet_io_run_mode mode, const int wrk_thread_num);
 
+/* destroy packet_io instance */
 void packet_io_instance_destroy(struct packet_io_instance *pinst);
 
+/*
+ * @brief open packet_io device for send/receive packets
+ *
+ * @param pinst:    packet_io instance pointer
+ * @param dev_name: packet_io device name
+ * @param nr_rxq:   number of receive queue for the device
+ * @param nr_txq:   number of send queue for the device
+**/
 struct packet_io_device *
-packet_io_open_device(struct packet_io_instance *instance, const char *dev_name, uint32_t nr_rxq, uint32_t nr_txq);
+packet_io_device_open(struct packet_io_instance *pinst, const char *dev_name, uint32_t nr_rxq, uint32_t nr_txq);
 
-void packet_io_close_device(struct packet_io_device *dev);
+/* close packet_io device */
+void packet_io_device_close(struct packet_io_device *dev);
 
-int packet_io_device_rx(struct packet_io_device *dev, uint32_t rx_queue_id, struct packet *p[], int nr_p);
+/*
+ * @brief packet_io device receive function
+ *
+ * @param pdev:   packet_io device pointer
+ * @param rxq_id: which queue will receive from
+ * @param p:      received packet's pointer array
+ * @param nr_p:   number of received packets
+**/
+int packet_io_device_rx(struct packet_io_device *pdev, uint32_t rxq_id, struct packet *p[], int nr_p);
 
-int packet_io_device_tx(struct packet_io_device *dev, uint32_t rx_queue_id, struct packet *p[], int nr_p);
\ No newline at end of file
+/*
+ * @brief packet_io device send function
+ *
+ * @param pdev:   packet_io device pointer
+ * @param rxq_id: which queue will send to
+ * @param p:      prepare to send packet's pointer array
+ * @param nr_p:   number of packets which prepare to send
+**/
+int packet_io_device_tx(struct packet_io_device *pdev, uint32_t txq_id, struct packet *p[], int nr_p);
\ No newline at end of file
diff --git a/src/packet_io/pcap_file_mode/pio_pcap_file.cpp b/src/packet_io/pcap_file_mode/pio_pcap_file.cpp
index fdd3c4f..aacc804 100644
--- a/src/packet_io/pcap_file_mode/pio_pcap_file.cpp
+++ b/src/packet_io/pcap_file_mode/pio_pcap_file.cpp
@@ -38,6 +38,6 @@ int pio_pcap_file_instance_create(struct packet_io_instance *pinst, int wrk_thre
     return 0;
 }
 
-void pio_pcap_file_instance_destroy(void) {
+void pio_pcap_file_instance_destroy(struct packet_io_instance *pinst) {
 
 }
\ No newline at end of file
diff --git a/src/packet_io/pcap_file_mode/pio_pcap_file.h b/src/packet_io/pcap_file_mode/pio_pcap_file.h
index 2906c7e..3dcf52c 100644
--- a/src/packet_io/pcap_file_mode/pio_pcap_file.h
+++ b/src/packet_io/pcap_file_mode/pio_pcap_file.h
@@ -47,4 +47,4 @@ int pio_pcap_file_device_send(struct packet_io_device *pdev, uint32_t txq_id, st
 
 int pio_pcap_file_instance_create(struct packet_io_instance *pinst, int wrk_thread_num);
 
-void pio_pcap_file_instance_destroy(void);
\ No newline at end of file
+void pio_pcap_file_instance_destroy(struct packet_io_instance *pinst);
\ No newline at end of file
diff --git a/src/packet_io/pcap_live_mode/pio_pcap_live.cpp b/src/packet_io/pcap_live_mode/pio_pcap_live.cpp
index 33af4db..a848ff0 100644
--- a/src/packet_io/pcap_live_mode/pio_pcap_live.cpp
+++ b/src/packet_io/pcap_live_mode/pio_pcap_live.cpp
@@ -35,6 +35,6 @@ int pio_pcap_live_instance_create(struct packet_io_instance *pinst, int wrk_thre
     return 0;
 }
 
-void pio_pcap_live_instance_destroy(void) {
+void pio_pcap_live_instance_destroy(struct packet_io_instance *pinst) {
 
 }
\ No newline at end of file
diff --git a/src/packet_io/pcap_live_mode/pio_pcap_live.h b/src/packet_io/pcap_live_mode/pio_pcap_live.h
index f6968c5..e71fac3 100644
--- a/src/packet_io/pcap_live_mode/pio_pcap_live.h
+++ b/src/packet_io/pcap_live_mode/pio_pcap_live.h
@@ -45,4 +45,4 @@ int pio_pcap_live_device_send(struct packet_io_device *pdev, uint32_t txq_id, st
 
 int pio_pcap_live_instance_create(struct packet_io_instance *pinst, int wrk_thread_num);
 
-void pio_pcap_live_instance_destroy(void);
\ No newline at end of file
+void pio_pcap_live_instance_destroy(struct packet_io_instance *pinst);
\ No newline at end of file
diff --git a/src/packet_io/test/gtest_packet_io.cpp b/src/packet_io/test/gtest_packet_io.cpp
index 6efcb71..7ff08f7 100644
--- a/src/packet_io/test/gtest_packet_io.cpp
+++ b/src/packet_io/test/gtest_packet_io.cpp
@@ -5,7 +5,7 @@
 TEST(PACKET_IO_Test, packet_io_open_device) {
 	struct packet_io_config ppio_config;
 	struct packet_io_instance *ppio_inst = packet_io_instance_create("stellar", PACKET_IO_RUN_MODE_PCAP_FILE, 2);
-  	EXPECT_EQ(packet_io_open_device(ppio_inst, nullptr, 1, 1), nullptr);
+  	EXPECT_EQ(packet_io_device_open(ppio_inst, nullptr, 1, 1), nullptr);
 }
 
 int main(int argc, char ** argv)