TSG-7425, sapp支持释放所有资源而不退出当前进程, 便于做单元测试.

24 files changed, 471 insertions, 202 deletions

diff --git a/bin/etc/sapp.toml b/bin/etc/sapp.toml
index 11c4f65..e4f08f8 100644
--- a/bin/etc/sapp.toml
+++ b/bin/etc/sapp.toml
@@ -104,7 +104,7 @@
     decrease_ratio="0.95"
     increase_ratio="1.005"
 ### note, unit of bypass_observe_time is second(s)     
-    recovery_observe_time=10
+    recovery_observe_time=30
 
 
 [PROTOCOL_FEATURE]
diff --git a/include/private/packet_io.h b/include/private/packet_io.h
index 3c26839..e188aee 100644
--- a/include/private/packet_io.h
+++ b/include/private/packet_io.h
@@ -111,6 +111,8 @@ typedef struct{
 	int (*dl_io_get_version)(void);

 	void * (*dl_io_device_alias)(unsigned int target_id, char *device_args);

 	int (*dl_io_send_only_thread_init)(void); /* 2020-06-01 lijia add, ������д����Ķ��������߳�, ��Ҫ���ô˽ӿڰ󶨵��ض�CPU������ */

+	void (*dl_io_free_send_handle)(int thread_seq, void *handle);

+	void (*dl_io_destroy)(void);

 }dl_io_fun_list_t;

 

 /* ���º����������, ������ȷ���������������� */

diff --git a/include/private/packet_io_internal.h b/include/private/packet_io_internal.h
index a0ae5c3..151787b 100644
--- a/include/private/packet_io_internal.h
+++ b/include/private/packet_io_internal.h
@@ -182,6 +182,8 @@ int packet_io_hook_forward(raw_pkt_t *raw_pkt, unsigned char dir, int thread_num
 int packet_io_hook_sendto(const raw_pkt_t *raw_pkt, unsigned char route_dir, char *send_pkt_data, void *send_pkt_io_reference);

 void sapp_pkt_dump_by_raw_pkt(int tid, const void *pkt, int pkt_len, int has_layer2_hdr, unsigned short eth_pro_host_order, enum _pkt_classify class_val);

 unsigned int dpdk_rte_ip_hash(const char *sipaddr, const char *dipaddr, int addrlen);

+void sendpacket_destroy(int tot_thread_num);

+void send_handle_destroy(int thread_index);

 

 

 /********************** funcitons for dynamic link ***********************/

@@ -236,9 +238,11 @@ extern int pcap_dl_io_get_version(void);
 extern void * pcap_dl_io_device_alias(unsigned int target_id, char *device_args);

 void packet_io_under_ddos_run(void);

 int packet_io_under_ddos_init(void);

+void packet_io_under_ddos_destroy(void);

 int packet_io_under_ddos_should_bypass(int thread_index);

 void update_under_ddos_stream_state(struct streaminfo *pstream);

 int packet_io_under_ddos_global_status(void);

+void pcap_dl_io_destroy(void);

 

 

 /********************** pcap funcitons for dynamic link ***********************/

@@ -267,6 +271,7 @@ extern int pag_dl_io_register_cb(PACKET_IO_CB_T fun);
 extern int pag_dl_io_register_exit_cb(PACKET_IO_EXIT_CB_T exit_fun); 

 extern int pag_dl_io_get_version(void);

 extern void * pag_dl_io_device_alias(unsigned int target_id, char *device_args);

+extern void pcap_dl_io_free_send_handle(int thread_seq, void *_handle);

 

 /********************** pag funcitons for dynamic link ***********************/

 

diff --git a/include/private/sapp_declaration.h b/include/private/sapp_declaration.h
index 3580161..bf1ead4 100644
--- a/include/private/sapp_declaration.h
+++ b/include/private/sapp_declaration.h
@@ -16,6 +16,11 @@ extern "C" {
 #define SAPP_MIN(a, b)       (((a) < (b)) ? (a) : (b))
 #endif
 
+#ifndef SAPP_FREE
+#define SAPP_FREE(m) do{if(m){free((void *)m); m = NULL;}}while(0)
+
+#endif
+
 
 /* ��Ƕ�ײ��̫��, ʹ�ü������ */
 #define ABBR_SAPP_LOG_LEVEL 	sapp_global_val->config.profiling.log.level
diff --git a/include/private/sapp_global_val.h b/include/private/sapp_global_val.h
index d272c41..3e06ad1 100644
--- a/include/private/sapp_global_val.h
+++ b/include/private/sapp_global_val.h
@@ -400,6 +400,7 @@ typedef struct{
 	sapp_gval_individual_fixed_fs_t field_stat2_para;
 	pthread_t thread_obtain_id[SAPP_MAX_THREADS]; /* pthread_create���ص�id */
 	pid_t thread_tid[SAPP_MAX_THREADS]; /* gdb �� top �鿴��id */
+	pthread_t thread_timer_event_id;
 	int cpu_bind_core_id_per_thread[SAPP_MAX_THREADS]; /* ���������bind_mask, ��¼ÿ��IO�����̰߳󶨵�cpu core id */
 	void *breakpad;
 	char overlay_layer_def[__ADDR_TYPE_MAX][SAPP_SUPPORT_LAYER_NUM_MAX]; /* ��ʾ��ԭʼ���Ļ����Ϸ�װ������, ��vxlan, ��Щ�㲻Ӧ�õ��ò��, ��עʱ��Ҫ���⴦�� */
diff --git a/include/private/stream_internal.h b/include/private/stream_internal.h
index 80d13ec..b5477da 100644
--- a/include/private/stream_internal.h
+++ b/include/private/stream_internal.h
@@ -435,6 +435,8 @@ void sapp_mem_free(void *data);
 void *sapp_mem_realloc(void *old_ptr, int size);
 void set_current_thread_cpu_affinity(int);
 int get_current_thread_cpu_affinity_id(void);
+void sapp_fs2_destroy(void);
+void sapp_gval_destroy(void);
 
 
 
diff --git a/include/public/stream_inc/stream_entry.h b/include/public/stream_inc/stream_entry.h
index 01ec865..9fda956 100644
--- a/include/public/stream_inc/stream_entry.h
+++ b/include/public/stream_inc/stream_entry.h
@@ -85,7 +85,8 @@ char POLLING_ENTRY(struct streaminfo *stream,  void **pme, int thread_seq,void *
 
 char PROT_PROCESS(stSessionInfo* session_info,  void **pme, int thread_seq,struct streaminfo *a_stream,const void *a_packet);
 
-int libsapp_setup_env(int argc, char *argv[]);
+int  libsapp_setup_env(int argc, char *argv[]);
+void libsapp_destroy_env(void);
 
 
 #ifdef __cplusplus
diff --git a/src/config/cmd_args.c b/src/config/cmd_args.c
index a74fc12..0a5f507 100644
--- a/src/config/cmd_args.c
+++ b/src/config/cmd_args.c
@@ -292,7 +292,7 @@ static int store_cmd_args(const char *opt_name, const char *opt_arg)
 	new_opt = (sapp_cmd_args_val_t *)sapp_mem_alloc(sizeof(sapp_cmd_args_val_t));
 	new_opt->long_opt_name = sapp_strdup(opt_name);
 	if(opt_arg){
-		new_opt->arg_val.string_value = strdup(opt_arg);
+		new_opt->arg_val.string_value = sapp_strdup(opt_arg);
 	}
 	
 	if(NULL == sapp_global_val->cla.cmd_args_array){
diff --git a/src/config/config_parse.cpp b/src/config/config_parse.cpp
index c19d250..3c68d7f 100644
--- a/src/config/config_parse.cpp
+++ b/src/config/config_parse.cpp
@@ -819,9 +819,11 @@ static int parse_asymmetric_addr_layer_config(void)
 				strcat(debug_layer_index_str, one_layer_index_str);
 			}
 		}
+#if 0		
 		sapp_runtime_log(RLOG_LV_DEBUG, "asymmetric_addr_layer: %s:%s", 
 					addr_type_abbreviation_ntop((enum addr_type_t)i),
 					debug_layer_index_str);
+#endif
 	}
 
 	fclose(fp);
@@ -1513,34 +1515,6 @@ static int config_sanity_check(void)
 }
 
 
-/* ��Ϊ����������,Ҫд�ļ�,log_handle�����ȳ�ʼ�� */
-static void log_handle_init(void)
-{
-	void *lhand;
-	MESA_handle_runtime_log_creation(ABBR_SAPP_LOG_CONF_FILE);
-	lhand = MESA_create_runtime_log_handle("./log/runtimelog", sapp_global_val->config.profiling.log.level);
-
-	sapp_global_val->individual_fixed.log_handle = lhand;
-}
-
-
-void sapp_gval_init(void)
-{
-	int tseq;
-	
-	sapp_global_val = (sapp_global_t *)calloc(1, sizeof(sapp_global_t));
-	sapp_global_val->individual_volatile = (sapp_gval_individual_volatile_t *)calloc(1, sizeof(sapp_gval_individual_volatile_t));
-
-	for(tseq = 0; tseq < SAPP_MAX_THREADS; tseq++){
-		sapp_global_val->mthread_volatile[tseq] = (sapp_gval_mthread_t *)calloc(1, sizeof(sapp_gval_mthread_t));
-		sapp_global_val->individual_fixed.cpu_bind_core_id_per_thread[tseq] = -1; /* ��ʼ������Ϊ-1, ��ʾû�󶨵��κ�һ��cpu core */
-	}
-
-	/* set default value, maybe not input by user or config file */
-	snprintf(sapp_global_val->config.packet_io.pcap_dumpfile_name, NAME_MAX, "dumpfile");
-
-	ABBR_CURRENT_TIME = time(NULL);
-}
 
 /*
 	֧�ֶ�д����ģʽ, ���¸��������ļ�·��, ���������ļ�·��.
@@ -1682,6 +1656,16 @@ int sapp_get_secondary_file_path(void)
 	return 0;
 }
 
+/* ��Ϊ����������,Ҫд�ļ�,log_handle�����ȳ�ʼ�� */
+static void log_handle_init(void)
+{
+	void *lhand;
+	MESA_handle_runtime_log_creation(ABBR_SAPP_LOG_CONF_FILE);
+	lhand = MESA_create_runtime_log_handle("./log/runtimelog", sapp_global_val->config.profiling.log.level);
+
+	sapp_global_val->individual_fixed.log_handle = lhand;
+}
+
 
 /* ��ȡ�����ļ�sapp.toml, ������ֻLoad, ���������ʹ����� */
 int sapp_parse_config(void)
@@ -2254,6 +2238,16 @@ done:
 	return final_ret;
 }
 
+
+
+void sapp_log_handle_destroy(void)
+{
+	//�˴�����, double free, �ݲ�ȷ�����ĵ�����
+	//MESA_destroy_runtime_log_handle(sapp_global_val->individual_fixed.log_handle);
+	sapp_global_val->individual_fixed.log_handle = NULL;
+}
+
+
 /* init breakpad_mini */
 void sapp_init_breakpad_mini(void)
 {
@@ -2274,6 +2268,8 @@ void sapp_segv_generate()
 	breakpad_segv_generate();
 }
 
+
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/config/sapp.toml.hex.array.c b/src/config/sapp.toml.hex.array.c
index a977583..f7a497b 100644
--- a/src/config/sapp.toml.hex.array.c
+++ b/src/config/sapp.toml.hex.array.c
@@ -407,7 +407,7 @@ const unsigned char _sapp_toml_sample_file_image[] = {
   0x73, 0x20, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x28, 0x73, 0x29, 0x20,
   0x20, 0x20, 0x20, 0x20, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x72, 0x65, 0x63,
   0x6f, 0x76, 0x65, 0x72, 0x79, 0x5f, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76,
-  0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x3d, 0x31, 0x30, 0x0a, 0x0a, 0x0a,
+  0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x3d, 0x33, 0x30, 0x0a, 0x0a, 0x0a,
   0x5b, 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x43, 0x4f, 0x4c, 0x5f, 0x46, 0x45,
   0x41, 0x54, 0x55, 0x52, 0x45, 0x5d, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x69,
   0x70, 0x76, 0x36, 0x5f, 0x64, 0x65, 0x63, 0x61, 0x70, 0x73, 0x75, 0x6c,
diff --git a/src/dealpkt/deal_gprs_tunnel.c b/src/dealpkt/deal_gprs_tunnel.c
index 49319ad..b9baf6f 100644
--- a/src/dealpkt/deal_gprs_tunnel.c
+++ b/src/dealpkt/deal_gprs_tunnel.c
@@ -156,9 +156,9 @@ static int get_gprs_hdr_len_v1(const struct gtp_hdr *gtph)
 	return base_len;
 }
 
-static int identify_gtp(const struct streaminfo *pstream,const char *payload, unsigned int payload_len)
+static int identify_gtp(const struct streaminfo *pstream,const char *payload, unsigned int udp_payload_len)
 {
-    if (payload_len >= sizeof(struct gtp_hdr))
+    if (udp_payload_len >= sizeof(struct gtp_hdr))
     {
         unsigned int gtp_u = (2152);
         unsigned int gtp_c = (2123);
@@ -186,7 +186,7 @@ static int identify_gtp(const struct streaminfo *pstream,const char *payload, un
             }else if(gtp_version == 1) {
                 unsigned short message_len = ntohs(gtp->len);
                 /* NOTE: gtp->len������gtpͷ���̶�����(8�ֽ�), ֻ������չͷ���͸��� */
-                if (message_len == (payload_len - sizeof(struct gtp_hdr))) {
+                if (message_len == (udp_payload_len - sizeof(struct gtp_hdr))) {
                     return 1;
                 }
             }else{
diff --git a/src/dealpkt/deal_udp.c b/src/dealpkt/deal_udp.c
index 64ad126..9e50386 100644
--- a/src/dealpkt/deal_udp.c
+++ b/src/dealpkt/deal_udp.c
@@ -668,18 +668,26 @@ int dealipv4udppkt(struct streamindex *pindex, const struct mesa_ip4_hdr * this_
 	}
 	
 	if(pstream_pr->stream_carry_up_layer_tunnel_type & STREAM_TUNNLE_TEREDO){
-		ret = teredo_entry(stack_stream_pr, udph, thread_num, routedir, raw_pkt, next_layer_offset);		
+		ret = teredo_entry(stack_stream_pr, udph, thread_num, routedir, raw_pkt, next_layer_offset);	
+		if(DROP == ret){
+			sapp_runtime_log(RLOG_LV_DEBUG, "UDP stream: %s, curdir:%d, return DROP.", printaddr(&pstream->addr, pstream->threadnum), pstream->curdir);
+			return ret;
+		}		
 	}
 	
 	if(pstream_pr->stream_carry_up_layer_tunnel_type & STREAM_TUNNLE_L2TP){
 		ret = l2tpv2_entry(stack_stream_pr, udph, thread_num, routedir, raw_pkt, next_layer_offset);	
+		if(DROP == ret){
+			sapp_runtime_log(RLOG_LV_DEBUG, "UDP stream: %s, curdir:%d, return DROP.", printaddr(&pstream->addr, pstream->threadnum), pstream->curdir);
+			return ret;
+		}			
 	}
+	
 	if(pstream_pr->stream_carry_up_layer_tunnel_type & STREAM_TUNNEL_GPRS_TUNNEL){
 		ret = gtp_entry(stack_stream_pr, udph, thread_num, routedir, raw_pkt, next_layer_offset);
 	}
 
 #endif
-
 	if(DROP == ret){
 		sapp_runtime_log(RLOG_LV_DEBUG, "UDP stream: %s, curdir:%d, return DROP.", printaddr(&pstream->addr, pstream->threadnum), pstream->curdir);
 	}
@@ -701,7 +709,7 @@ int dealipv6udppkt(struct streamindex *pindex,const struct mesa_ip6_hdr *a_packe
 	sapp_gval_mthread_sys_stat_t *local_sys_stat = &sapp_global_val->mthread_volatile[thread_num]->sys_stat;
 	int try_to_update_addr_info = 0;
 	sapp_dup_pkt_key_v6_t dup_bloom_key;
-
+	int next_layer_offset;
 	iplen = ntohs (a_packet->ip6_payload_len);
 	ulen = ntohs (udph->uh_ulen);
 
@@ -740,6 +748,7 @@ int dealipv6udppkt(struct streamindex *pindex,const struct mesa_ip6_hdr *a_packe
 	pstream->type=STREAM_TYPE_UDP;
 	pstream->routedir=routedir;
 	pstream->threadnum=thread_num;
+	next_layer_offset = offset_to_raw_pkt_hdr + sizeof(struct mesa_udp_hdr);
 
 	//add by lqy 20130530 
 	// udp�в���������ԭ�Ĺ�������ֱ�Ӱ�������������
@@ -859,6 +868,18 @@ int dealipv6udppkt(struct streamindex *pindex,const struct mesa_ip6_hdr *a_packe
 		udp_cleardata(pstream);
 	}
 
+	if(pstream_pr->stream_carry_up_layer_tunnel_type & STREAM_TUNNLE_L2TP){
+		ret = l2tpv2_entry(pstream_pr, udph, thread_num, routedir, raw_pkt, next_layer_offset);	
+		if(DROP == ret){
+			sapp_runtime_log(RLOG_LV_DEBUG, "UDP stream: %s, curdir:%d, return DROP.", printaddr(&pstream->addr, pstream->threadnum), pstream->curdir);
+			return ret;
+		}		
+	}
+
+	if(pstream_pr->stream_carry_up_layer_tunnel_type & STREAM_TUNNEL_GPRS_TUNNEL){
+		ret = gtp_entry(pstream_pr, udph, thread_num, routedir, raw_pkt, next_layer_offset);
+	}
+
 	/* TODO 2, ����IPv6��UDP port 1701 L2TP�� */
 
 	if(DROP == ret){
diff --git a/src/dealpkt/stream_manage.c b/src/dealpkt/stream_manage.c
index 5e9e0c8..38aef34 100644
--- a/src/dealpkt/stream_manage.c
+++ b/src/dealpkt/stream_manage.c
@@ -36,7 +36,7 @@ int sapp_version_v4_20191119;
 	����İ汾������packet_io_libУ��, ��ֹƽ̨��dl.so��ƥ�䵼�������쳣, �����޸���BUGδ����.
 */
 #if IOMODE_PCAP
-int sapp_packet_io_v = 20190820;
+int sapp_packet_io_v = 20210907;
 #elif IOMODE_PAG
 int sapp_packet_io_v = 20151106;
 #elif IOMODE_PFRING
diff --git a/src/inner_plug/g_device_plug.cpp b/src/inner_plug/g_device_plug.cpp
index 7bf791e..3ef5507 100644
--- a/src/inner_plug/g_device_plug.cpp
+++ b/src/inner_plug/g_device_plug.cpp
@@ -30,7 +30,7 @@ extern void (*ptr_marsio_buff_free)(struct mr_instance * instance, marsio_buff_t
  __attribute__((__used__)) const char *g_device_plug_git_ver = GIT_VERSION;
 #endif
 
-#define MAX_SUPPORT_GDEV_NUM	(16)  /* ÿ���߳������֧��inline device������, ���ݷ������Ժ��߳����IJ�ͬ, ��Сֵ��MAX_SUPPORT_GDEV_NUM, ���ֵ��MAX_SUPPORT_GDEV_NUM * sapp_thread_count */
+#define MAX_SUPPORT_GDEV_NUM	(64)  /* ÿ���߳������֧��inline device������, ���ݷ������Ժ��߳����IJ�ͬ, ��Сֵ��MAX_SUPPORT_GDEV_NUM, ���ֵ��MAX_SUPPORT_GDEV_NUM * sapp_thread_count */
 #define MAX_VXLAN_SERVICE_NUM	(256) /* Ŀǰ���֧��255�ֲ�ͬҵ��, �����±������255 */
 
 #define MAX_NO_PKT_TIMEOUT 		(120) /* ������ô��ʱ��һֱû�����, �Ѵ��豸�޳� */
@@ -985,8 +985,12 @@ int gdev_keepalive_plug_init(void)
 	char deploy_mode_string[32];
 	int opt_len;
 	int i;
-
+#if 0
 	gdev_kp_log_handle = MESA_create_runtime_log_handle("./log/gdev_keeplive.log", 10);
+#else
+	/* 2020-12-23 ����������, �˴�����sapp��log_handle, ��module��������, ���ٵ���д�ļ��� */
+	gdev_kp_log_handle = sapp_global_val->individual_fixed.log_handle;
+#endif
 
 #ifdef GIT_VERSION
 	MESA_handle_runtime_log(gdev_kp_log_handle, 30, "[gdev_keepalive]", "g_device_plug version: %s\n", g_device_plug_git_ver);	
diff --git a/src/inner_plug/sapp_assistant.cpp b/src/inner_plug/sapp_assistant.cpp
index 4eaf34d..c55f87d 100644
--- a/src/inner_plug/sapp_assistant.cpp
+++ b/src/inner_plug/sapp_assistant.cpp
@@ -1051,6 +1051,34 @@ int sapp_assistant_init(void)
 }
 #endif
 
+static int sapp_line_protocol_init(void)
+{
+	log_line_protocol_handle_t *lp_handle;
+
+	if(0 == sapp_global_val->config.profiling.log.remote.enabled){
+		sapp_log(RLOG_LV_INFO, 10, 10, "profiling.log.remote.enabled = 0, line protocol is disable.\n");
+		return 0;
+	}
+
+	if(strncasecmp(sapp_global_val->config.profiling.log.remote.remote_send_out_type, "line_protocol", strlen("line_protocol")) != 0){
+		sapp_log(RLOG_LV_INFO, 10, 10, "profiling.log.remote.enabled, remote_send_out_type is field_stat2, line protocol is disable.\n");
+		return 0;
+	}
+
+	lp_handle = (log_line_protocol_handle_t *)calloc(1, sizeof(log_line_protocol_handle_t));
+
+	lp_handle->sock_fd = socket(AF_INET, SOCK_DGRAM, 0);
+    lp_handle->sock_addr.sin_family = AF_INET;
+    inet_pton(AF_INET, sapp_global_val->config.profiling.log.remote.server_ip, &lp_handle->sock_addr.sin_addr.s_addr);
+    lp_handle->sock_addr.sin_port = ntohs((unsigned short)sapp_global_val->config.profiling.log.remote.server_port);
+
+    lp_handle->last_send_metrics_time = 0;
+
+	sapp_global_val->individual_fixed.profiling_log_remote_handle = lp_handle;
+	
+	return 0;
+}
+
 
 static int fs2_plug_entry_historgram_init(void *fs2_handle)
 {
@@ -1265,31 +1293,12 @@ static int sapp_fs2_init(void)
 }
 
 
-static int sapp_line_protocol_init(void)
+void sapp_fs2_destroy(void)
 {
-	log_line_protocol_handle_t *lp_handle;
-
-	if(0 == sapp_global_val->config.profiling.log.remote.enabled){
-		sapp_log(RLOG_LV_INFO, 10, 10, "profiling.log.remote.enabled = 0, line protocol is disable.\n");
-		return 0;
+	if(sapp_global_val->individual_fixed.field_stat2_para.field_stat2_handle){
+		FS_stop(&sapp_global_val->individual_fixed.field_stat2_para.field_stat2_handle);
+		sapp_global_val->individual_fixed.field_stat2_para.field_stat2_handle = NULL;
 	}
-
-	if(strncasecmp(sapp_global_val->config.profiling.log.remote.remote_send_out_type, "line_protocol", strlen("line_protocol")) != 0){
-		sapp_log(RLOG_LV_INFO, 10, 10, "profiling.log.remote.enabled, remote_send_out_type is field_stat2, line protocol is disable.\n");
-		return 0;
-	}
-
-	lp_handle = (log_line_protocol_handle_t *)calloc(1, sizeof(log_line_protocol_handle_t));
-
-	lp_handle->sock_fd = socket(AF_INET, SOCK_DGRAM, 0);
-    lp_handle->sock_addr.sin_family = AF_INET;
-    inet_pton(AF_INET, sapp_global_val->config.profiling.log.remote.server_ip, &lp_handle->sock_addr.sin_addr.s_addr);
-    lp_handle->sock_addr.sin_port = ntohs((unsigned short)sapp_global_val->config.profiling.log.remote.server_port);
-
-    lp_handle->last_send_metrics_time = 0;
-
-	sapp_global_val->individual_fixed.profiling_log_remote_handle = lp_handle;
-	
-	return 0;
 }
 
+
diff --git a/src/packet_io/packet_io.c b/src/packet_io/packet_io.c
index 5c18e3b..6c54223 100644
--- a/src/packet_io/packet_io.c
+++ b/src/packet_io/packet_io.c
@@ -730,89 +730,12 @@ static int mesa_default_pkt_cb(const raw_pkt_t *p_raw_pkt, unsigned char dir, in
 }
 
 
-void packet_io_exit(void)
-{
-	/* ����ģʽ��, �������߳�һЩʱ��, ����δ�������, ����MESA_tcp�첽�������ݵȵ�. */
-	sapp_set_current_state(SAPP_STATE_READY_TO_EXIT);
-	sleep(sapp_global_val->config.packet_io.dumpfile_sleep_time_before_exit);
-	plugctrl_destroy_all_plug();
-	
-	exit(0);
-	
-	return;
-}
-
-int packet_io_init(int argc, char *argv[])
-{
-	int i, ret;
-
-	if(sapp_global_val->config.stream.tcp.inject.remedy_kill_tcp_by_inline_device != 0){
-		ret = gdev_block_init();
-		if(ret < 0){
-			printf("\n\033[33m[Error]gdev_block_init error!\033[0m\n");
-			return -1;
-		}
-	}
-		
-	if(0 == use_custom_pkt_cb){
-		packet_io_register_cb(mesa_default_pkt_cb);	
-	}
-
-	if(NULL == dl_io_fun_list.dl_io_init){
-		printf("Error, packet io library must support 'dl_io_init' !\n");
-		exit(0);
-	}
-	
-	if(dl_io_fun_list.dl_io_init(argc, argv) < 0){
-		return -1;
-	}
-
-	for(i = 0; i < MAX_THREAD_NUM; i++){
-		g_send_buf_pool[i] = (UINT8 *)malloc(SENDPACKET_BUF_LEN);
-	}
-
-	if(sendpacket_init_new(g_packet_io_thread_num) < 0){
-		printf("Error, sendpacket_init_new error !\n");
-		return -1;
-	}
-
-#if 0 /* 20161117 lijia move to send_handle_init() */
-	for(i = 0; i < g_packet_io_thread_num; i++){
-		g_packet_dl_send_handle[i] = dl_io_fun_list.dl_io_get_send_handle(i);
-	}
-#endif
-
-	if(CAP_MODEL_PCAP_DUMPFILE == g_packet_io_cap_mode){
-		dl_io_fun_list.dl_io_register_exit_cb(packet_io_exit);
-	}
-
-	if(sapp_global_val->config.packet_io.inject_pkt_mode == SEND_PKT_MODE_GDEV){
-    	parse_send_gdev_ip_conf("conf/send_gdev.conf");
-    }
-
-	if(sapp_global_val->config.packet_io.inject_pkt_mode == SEND_PKT_MODE_STACK_2_LAYER_MUTI_ROUTE){
-		parse_send_route_conf("conf/send_route.conf");
-	}
-	
-	packet_io_device_alias_init();
-
-	packet_io_status_init();
-
-	if(packet_io_under_ddos_init() < 0){
-		return -1;
-	}
-
-	return 0;
-}
-
 extern int g_StreamTcpAllFunNum;
 extern int g_Ipv6FunNum;
 extern  int app_function_rationality_check(void);
  /* not return until error, or use 'CAP_MODEL_PCAP_DUMPFILE' mode */
 void packet_io_run(void)
-{
-	pthread_t pid;
-		
+{		
 	if(NULL == dl_io_fun_list.dl_io_run){
 		printf("Error, packet io library must support 'dl_io_run' !\n");
 		sapp_runtime_log(RLOG_LV_FATAL, "%s:%d: Error, packet io library must support 'dl_io_run' !\n", __FILE__, __LINE__);
@@ -821,7 +744,7 @@ void packet_io_run(void)
 
 	app_function_rationality_check();
 	
-	pthread_create(&pid, NULL, sapp_time_event_thread ,NULL);
+	pthread_create(&sapp_global_val->individual_fixed.thread_timer_event_id, NULL, sapp_time_event_thread ,NULL);
 
 	packet_io_under_ddos_run();
 	
@@ -829,14 +752,7 @@ void packet_io_run(void)
 }
 
 extern volatile int update_packet_io_status_sw;
-void packet_io_clean_thread_context(int thread_seq)
-{
-	free_thread_stream(thread_seq);
 
-	ipv4_frag_per_thread_exit(thread_seq);
-	ipv6_frag_per_thread_exit(thread_seq);
-	pptp_per_thread_exit(thread_seq);
-}
 
 
 MESA_send_handle *packet_io_get_send_handle(int thread_id)
@@ -1385,6 +1301,116 @@ int packet_io_send_raw(int thread_num, char *data, int datalen,unsigned int targ
 	return ret;
 }
 
+
+
+void packet_io_clean_thread_context(int thread_seq)
+{
+	free_thread_stream(thread_seq);
+
+	ipv4_frag_per_thread_exit(thread_seq);
+	ipv6_frag_per_thread_exit(thread_seq);
+	pptp_per_thread_exit(thread_seq);
+}
+
+void packet_io_lib_destroy(void)
+{
+	if(dl_io_fun_list.dl_io_destroy){
+		dl_io_fun_list.dl_io_destroy();
+	}
+}
+
+int packet_io_init(int argc, char *argv[])
+{
+	int i, ret;
+
+	if(sapp_global_val->config.stream.tcp.inject.remedy_kill_tcp_by_inline_device != 0){
+		ret = gdev_block_init();
+		if(ret < 0){
+			printf("\n\033[33m[Error]gdev_block_init error!\033[0m\n");
+			return -1;
+		}
+	}
+		
+	if(0 == use_custom_pkt_cb){
+		packet_io_register_cb(mesa_default_pkt_cb);	
+	}
+
+	if(NULL == dl_io_fun_list.dl_io_init){
+		printf("Error, packet io library must support 'dl_io_init' !\n");
+		exit(0);
+	}
+	
+	if(dl_io_fun_list.dl_io_init(argc, argv) < 0){
+		return -1;
+	}
+
+	for(i = 0; i < MAX_THREAD_NUM; i++){
+		g_send_buf_pool[i] = (UINT8 *)malloc(SENDPACKET_BUF_LEN);
+	}
+
+	if(sendpacket_init_new(g_packet_io_thread_num) < 0){
+		printf("Error, sendpacket_init_new error !\n");
+		return -1;
+	}
+
+#if 0 /* 20161117 lijia move to send_handle_init() */
+	for(i = 0; i < g_packet_io_thread_num; i++){
+		g_packet_dl_send_handle[i] = dl_io_fun_list.dl_io_get_send_handle(i);
+	}
+#endif
+
+	if(CAP_MODEL_PCAP_DUMPFILE == g_packet_io_cap_mode){
+		dl_io_fun_list.dl_io_register_exit_cb(libsapp_destroy_env);
+	}
+
+	if(sapp_global_val->config.packet_io.inject_pkt_mode == SEND_PKT_MODE_GDEV){
+    	parse_send_gdev_ip_conf("conf/send_gdev.conf");
+    }
+
+	if(sapp_global_val->config.packet_io.inject_pkt_mode == SEND_PKT_MODE_STACK_2_LAYER_MUTI_ROUTE){
+		parse_send_route_conf("conf/send_route.conf");
+	}
+	
+	packet_io_device_alias_init();
+
+	packet_io_status_init();
+
+	if(packet_io_under_ddos_init() < 0){
+		return -1;
+	}
+
+	return 0;
+}
+
+
+void packet_io_exit(void)
+{
+	/* ����ģʽ��, ��������IO�߳�һЩʱ��, ����δ�������, �����첽������־, field_statˢ��ͳ�����ݵȵ�. */
+	sapp_set_current_state(SAPP_STATE_READY_TO_EXIT);
+	sleep(sapp_global_val->config.packet_io.dumpfile_sleep_time_before_exit);
+
+	sysinfo_output(); /* ���һ��дͳ����Ϣ, �رձ�־λ, ��Ϊ�����������, ��д��û������ */
+	update_packet_io_status_sw = 0;
+		
+	plugctrl_destroy_all_plug();
+	sendpacket_destroy(g_packet_io_thread_num);
+	packet_io_lib_destroy();
+
+
+	sapp_log_handle_destroy();
+	sapp_fs2_destroy();
+	packet_io_under_ddos_destroy();
+
+	sapp_timer_destroy(sapp_global_val->individual_fixed.sapp_standalone_timer);
+	
+	sapp_gval_destroy();
+
+	exit(0);
+	
+	return;
+}
+
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/packet_io/packet_io_lib_load.c b/src/packet_io/packet_io_lib_load.c
index 5aedc73..6ad5861 100644
--- a/src/packet_io/packet_io_lib_load.c
+++ b/src/packet_io/packet_io_lib_load.c
@@ -93,6 +93,8 @@ static int packet_io_lib_load_by_mode(int cap_mode, const char *full_lib_path)
 	dl_io_fun_list.dl_io_get_version = (int (*)(void))packet_io_symbol_load_by_mode(io_lib_handle, cap_mode, "dl_io_get_version");
 	dl_io_fun_list.dl_io_device_alias = (void * (*)(unsigned int , char *))packet_io_symbol_load_by_mode(io_lib_handle, cap_mode, "dl_io_device_alias");
 
+	dl_io_fun_list.dl_io_free_send_handle = (void (*dl_io_free_send_handle)(int thread_seq, void *handle))packet_io_symbol_load_by_mode(io_lib_handle, cap_mode, "dl_io_free_send_handle");
+	dl_io_fun_list.dl_io_destroy = (void (*dl_io_destroy)(void))packet_io_symbol_load_by_mode(io_lib_handle, cap_mode, "dl_io_destroy");
 	return 0;
 }
 #else
@@ -102,30 +104,30 @@ static int packet_io_lib_load_by_mode(int cap_mode, const char *no_use)
 {
 #ifdef IOMODE_PCAP
 	if((CAP_MODEL_PCAP_ONLINE == cap_mode) || (CAP_MODEL_PCAP_DUMPFILE == cap_mode)){
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_set_cap_level = (int (*)(int cap_level))pcap_dl_io_set_cap_level;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_set_cap_mode = (int (*)(int cap_mode))pcap_dl_io_set_cap_mode;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_set_topology_mode = (int (*)(int topology_mode))pcap_dl_io_set_topology_mode;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_set_capdev_parallel = (int (*)(const char *cap_dev))pcap_dl_io_set_capdev_parallel;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_set_capdev_serial = (int (*)(const char *up_dev, const char *down_dev))pcap_dl_io_set_capdev_serial;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_set_capture_filter = (int (*)(const char *filter_rule))pcap_dl_io_set_capture_filter;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_set_cap_buf_queue = (int (*)(int queue_num_max))pcap_dl_io_set_cap_buf_queue;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_set_work_thread_num = (int (*)(int thread_num_max))pcap_dl_io_set_work_thread_num;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_get_app_drop_num = (long (*)(int thread_num))pcap_dl_io_get_app_drop_num;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_get_lib_drop_num = (long (*)(void))pcap_dl_io_get_lib_drop_num;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_get_sendbuf = (unsigned char *  (*)(void *, int thread_num))pcap_dl_io_get_sendbuf;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_free_sendbuf = (void (*)(void *, int thread_num))pcap_dl_io_free_sendbuf;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_get_send_handle = (void * (*)(int thread_num))pcap_dl_io_get_send_handle;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_low_level_send = (int (*)(void *, unsigned char *,int ,int , int ,int ,const raw_pkt_t *))pcap_dl_io_low_level_send;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_raw_pkt_send = (int (*)(void *, unsigned char *,int , void *,int,unsigned char, const raw_pkt_t * ))pcap_dl_io_raw_pkt_send;	
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_init = (int (*)(int argc, char *argv[]))pcap_dl_io_init;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_run = (void (*)(void))pcap_dl_io_run;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_register_cb = (int (*)(PACKET_IO_CB_T fun))pcap_dl_io_register_cb;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_register_exit_cb = (int (*)(PACKET_IO_EXIT_CB_T exit_fun))pcap_dl_io_register_exit_cb;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_get_version = (int (*)(void))pcap_dl_io_get_version;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_device_alias = (void * (*)(unsigned int , char *))pcap_dl_io_device_alias;
-		dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE].dl_io_send_only_thread_init = NULL;
-
-		memcpy(&dl_io_fun_list_static_link[CAP_MODEL_PCAP_DUMPFILE], &dl_io_fun_list_static_link[CAP_MODEL_PCAP_ONLINE], sizeof(dl_io_fun_list_t));
+		dl_io_fun_list_static_link[cap_mode].dl_io_set_cap_level = (int (*)(int cap_level))pcap_dl_io_set_cap_level;
+		dl_io_fun_list_static_link[cap_mode].dl_io_set_cap_mode = (int (*)(int cap_mode))pcap_dl_io_set_cap_mode;
+		dl_io_fun_list_static_link[cap_mode].dl_io_set_topology_mode = (int (*)(int topology_mode))pcap_dl_io_set_topology_mode;
+		dl_io_fun_list_static_link[cap_mode].dl_io_set_capdev_parallel = (int (*)(const char *cap_dev))pcap_dl_io_set_capdev_parallel;
+		dl_io_fun_list_static_link[cap_mode].dl_io_set_capdev_serial = (int (*)(const char *up_dev, const char *down_dev))pcap_dl_io_set_capdev_serial;
+		dl_io_fun_list_static_link[cap_mode].dl_io_set_capture_filter = (int (*)(const char *filter_rule))pcap_dl_io_set_capture_filter;
+		dl_io_fun_list_static_link[cap_mode].dl_io_set_cap_buf_queue = (int (*)(int queue_num_max))pcap_dl_io_set_cap_buf_queue;
+		dl_io_fun_list_static_link[cap_mode].dl_io_set_work_thread_num = (int (*)(int thread_num_max))pcap_dl_io_set_work_thread_num;
+		dl_io_fun_list_static_link[cap_mode].dl_io_get_app_drop_num = (long (*)(int thread_num))pcap_dl_io_get_app_drop_num;
+		dl_io_fun_list_static_link[cap_mode].dl_io_get_lib_drop_num = (long (*)(void))pcap_dl_io_get_lib_drop_num;
+		dl_io_fun_list_static_link[cap_mode].dl_io_get_sendbuf = (unsigned char *  (*)(void *, int thread_num))pcap_dl_io_get_sendbuf;
+		dl_io_fun_list_static_link[cap_mode].dl_io_free_sendbuf = (void (*)(void *, int thread_num))pcap_dl_io_free_sendbuf;
+		dl_io_fun_list_static_link[cap_mode].dl_io_get_send_handle = (void * (*)(int thread_num))pcap_dl_io_get_send_handle;
+		dl_io_fun_list_static_link[cap_mode].dl_io_low_level_send = (int (*)(void *, unsigned char *,int ,int , int ,int ,const raw_pkt_t *))pcap_dl_io_low_level_send;
+		dl_io_fun_list_static_link[cap_mode].dl_io_raw_pkt_send = (int (*)(void *, unsigned char *,int , void *,int,unsigned char, const raw_pkt_t * ))pcap_dl_io_raw_pkt_send;	
+		dl_io_fun_list_static_link[cap_mode].dl_io_init = (int (*)(int argc, char *argv[]))pcap_dl_io_init;
+		dl_io_fun_list_static_link[cap_mode].dl_io_run = (void (*)(void))pcap_dl_io_run;
+		dl_io_fun_list_static_link[cap_mode].dl_io_register_cb = (int (*)(PACKET_IO_CB_T fun))pcap_dl_io_register_cb;
+		dl_io_fun_list_static_link[cap_mode].dl_io_register_exit_cb = (int (*)(PACKET_IO_EXIT_CB_T exit_fun))pcap_dl_io_register_exit_cb;
+		dl_io_fun_list_static_link[cap_mode].dl_io_get_version = (int (*)(void))pcap_dl_io_get_version;
+		dl_io_fun_list_static_link[cap_mode].dl_io_device_alias = (void * (*)(unsigned int , char *))pcap_dl_io_device_alias;
+		dl_io_fun_list_static_link[cap_mode].dl_io_send_only_thread_init = NULL;
+		dl_io_fun_list_static_link[cap_mode].dl_io_free_send_handle = pcap_dl_io_free_send_handle;
+		dl_io_fun_list_static_link[cap_mode].dl_io_destroy = pcap_dl_io_destroy;
 	}
 #endif
 
@@ -153,6 +155,8 @@ static int packet_io_lib_load_by_mode(int cap_mode, const char *no_use)
 		dl_io_fun_list_static_link[CAP_MODEL_MARSIOV4].dl_io_get_version = (int (*)(void))marsio_dl_io_get_version;
 		dl_io_fun_list_static_link[CAP_MODEL_MARSIOV4].dl_io_device_alias = (void * (*)(unsigned int , char *))marsio_dl_io_device_alias;
 		dl_io_fun_list_static_link[CAP_MODEL_MARSIOV4].dl_io_send_only_thread_init = (int (*)(void))marsio_dl_io_send_only_thread_init;
+		dl_io_fun_list_static_link[CAP_MODEL_MARSIOV4].dl_io_free_send_handle = NULL;
+		dl_io_fun_list_static_link[CAP_MODEL_MARSIOV4].dl_io_destroy = NULL;
 	}
 #endif 
 
@@ -295,7 +299,8 @@ int packet_io_lib_load(int cap_mode)
 	__print_critical_msg((void *)dl_io_fun_list.dl_io_register_cb, full_lib_path, "dl_io_register_cb");
 	__print_err_msg((void *)dl_io_fun_list.dl_io_register_exit_cb, full_lib_path, "dl_io_register_exit_cb");
 	__print_err_msg((void *)dl_io_fun_list.dl_io_get_version, full_lib_path, "dl_io_get_version");
-	__print_err_msg((void *)dl_io_fun_list.dl_io_device_alias, full_lib_path, "dl_io_device_alias");
+	__print_err_msg((void *)dl_io_fun_list.dl_io_free_send_handle, full_lib_path, "dl_io_free_send_handle");
+	__print_err_msg((void *)dl_io_fun_list.dl_io_destroy, full_lib_path, "dl_io_destroy");
 
 #if LINK_MODE_DYNAMIC
 	if(dl_io_fun_list.dl_io_get_version != NULL){
diff --git a/src/packet_io/packet_io_pcap.c b/src/packet_io/packet_io_pcap.c
index b2e237d..ecdd816 100644
--- a/src/packet_io/packet_io_pcap.c
+++ b/src/packet_io/packet_io_pcap.c
@@ -11,7 +11,7 @@ extern "C" {
 #define PROCESS_BAR_SW 		(1)

 

 int g_pcap_version_VERSION_20210722; 

-static int g_pcap_version_internal = 20210722;

+static int g_pcap_version_internal = 20210907;

 

 #define PCAP_SNAPLEN_MAX	(65535)

 

@@ -560,7 +560,7 @@ void pcap_dl_io_free_sendbuf(void *phandle, int thread_num)
 	return;

 }

 

-void *pcap_dl_io_get_send_handle(int thread_num)

+void *pcap_dl_io_get_send_handle(int thread_seq)

 {

 	pcap_send_handle *send_handle = (pcap_send_handle *)calloc(1, sizeof(pcap_send_handle));

 

@@ -569,7 +569,7 @@ void *pcap_dl_io_get_send_handle(int thread_num)
 		printf("socket error: %s\n", strerror(errno));

 		goto err_exit;

 	}

-	send_handle->thread_num = thread_num;

+	send_handle->thread_num = thread_seq;

 	strncpy(send_handle->saddr_up.sa_data, g_pcap_up_dev_name, sizeof(struct sockaddr));

 	strncpy(send_handle->saddr_down.sa_data, g_pcap_down_dev_name, sizeof(struct sockaddr));

 	send_handle->saddr_up.sa_family = PF_INET;

@@ -582,6 +582,22 @@ err_exit:
 	return NULL;

 }

 

+void pcap_dl_io_free_send_handle(int thread_seq, void *_handle)

+{

+	pcap_send_handle *send_handle = (pcap_send_handle *)_handle;

+

+	if(NULL == _handle){

+		return;

+	}

+

+	if(send_handle->raw_eth_fd > 0){

+		close(send_handle->raw_eth_fd);

+		send_handle->raw_eth_fd = -1;

+	}

+

+	free(_handle);

+}

+

 int pcap_dl_io_get_version(void)

 {

 	return g_pcap_version_internal;

@@ -600,17 +616,21 @@ int pcap_dl_io_raw_pkt_send(void *phandle, unsigned char *data,int datalen, void
 	snd_addr.sa_family = PF_INET;

 

 /* 2017-02-28 lijia add, for tcp offload sendback, ijЩ���߰����������, GRO, GSO�����Թز���, ���´���ģʽ������� */

+	//TODO, �˴�����Ҫ�Ƶ�sapp���ϲ㷢���ӿ�, ������ô�ײ�Ľӿ���offload, ��Ϊ�������漰��marsioģʽ

+    //���ݷ���������MTU�Զ��ж�, �Ƿ���Ҫoffload.

+#if 0 

 	if(datalen > 1514){

 		if(sapp_global_single.cfg_send_tcp_offload_sw){

-			sapp_runtime_log(10, "send pkt len > 1514,  send packet by tcp_offload,  the slice pkt maybe different with raw pkt !\n");

+			sapp_runtime_log(10, "send pkt len is:%d, more than 1514,  send packet by tcp_offload,  the slice pkt maybe different with raw pkt !\n", datalen);

 			ret = sendpacket_tcp_offload((char *)data, datalen, send_handle->raw_eth_fd, (struct sockaddr *)&snd_addr);

 			return ret;		

 		}

 

-		sapp_runtime_log(30, "send pkt len > 1514,  but main.conf->send_tcp_offload is disable !\n");

+		sapp_runtime_log(30, "send pkt len is:%d, moren than 1514,  but main.conf->send_tcp_offload is disable !\n", datalen);

 		return -1;

 	}

-	

+#endif

+

 send_again:

 	ret = sendto(send_handle->raw_eth_fd, data, datalen, 0, (struct sockaddr *)&snd_addr, sizeof(struct sockaddr));

 	if(ret < 0){

@@ -775,7 +795,7 @@ static void *__pcap_work_thread(void *arg)
 	*/

 	int polling_enabled = sapp_global_val->config.packet_io.polling_enabled;

 

-	while(1){

+	while(SAPP_STATE_PROCESSING == sapp_get_current_state()){

 		sapp_global_val->mthread_volatile[thread_num]->sys_stat.count[SAPP_STAT_FETCH_PKT]++;

 		

 		if(0 == MESA_lqueue_get_count(work_thread_pool[thread_num].pkt_queue)){

@@ -845,9 +865,6 @@ static void *__pcap_work_thread(void *arg)
 	}

 

 exit:

-	while(1){

-		pause();

-	}

 	return NULL;

 }

 

@@ -1356,9 +1373,7 @@ static void *pcap_io_thread(void *arg)
 		}else if(pcap_ret < 0){

 			printf("pcap err, %s\n", pcap_geterr(handle));

 			sapp_set_current_state(SAPP_STATE_READY_TO_EXIT);

-			sched_yield();

-			sleep(1);

-			exit(1);

+			exit(0);

 		}

 	}

 

@@ -1485,6 +1500,18 @@ int pcap_dl_io_init(int argc, char *argv[])
 	return 0;

 }

 

+void pcap_dl_io_destroy(void)

+{

+	if(pcap_up_handle){

+		pcap_close(pcap_up_handle);

+		pcap_up_handle = NULL;

+	}

+	if(pcap_down_handle){

+		pcap_close(pcap_down_handle);

+		pcap_down_handle = NULL;

+	}	

+}

+

 /* not return except some error occur. */

 void pcap_dl_io_run(void)

 {

diff --git a/src/packet_io/sendpacket.c b/src/packet_io/sendpacket.c
index 1aaff81..003d143 100644
--- a/src/packet_io/sendpacket.c
+++ b/src/packet_io/sendpacket.c
@@ -3206,7 +3206,7 @@ static int skip_asymmetric_presence_layer(const struct streaminfo *stream, UCHAR
 
 static int get_eth_carry_layer_type(struct streaminfo *stream, UCHAR send_stream_dir)
 {
-	int ret = stream->addr.addrtype;
+	int ret = -1;
 	struct streaminfo_private *stream_pr = (struct streaminfo_private *)stream;
 
 	while(stream_pr && stream_pr->pfather_pr){
@@ -3234,9 +3234,9 @@ static int get_eth_carry_layer_type(struct streaminfo *stream, UCHAR send_stream
 				}			
 			}
 		}
-
-		ret = stream->addr.addrtype;	
+	
 		stream_pr = stream_pr->pfather_pr;
+		stream = &stream_pr->stream_public;
 	}
 
 	return ret;
@@ -3407,12 +3407,12 @@ static int send_handle_init(int tot_thread_num)
 #endif	
 		g_send_handle[i].raw_ipv4_fd = socket(PF_INET, SOCK_RAW, IPPROTO_RAW);
 		if(g_send_handle[i].raw_ipv4_fd < 0){
-			sapp_log(30, ~0, ~0, "socket v4 error, %s\n", strerror(errno));
+			sapp_log(30, ~0, ~0, "create raw socket v4 error, %s\n", strerror(errno));
 			return -1;
 		}
 		g_send_handle[i].raw_udp_fd = socket(AF_INET, SOCK_DGRAM|SOCK_CLOEXEC, 0);
 		if(g_send_handle[i].raw_udp_fd < 0){
-			sapp_log(30, ~0, ~0, "socket udp error, %s\n", strerror(errno));
+			sapp_log(30, ~0, ~0, "create udp socket error, %s\n", strerror(errno));
 			return -1;
 		} 
 
@@ -3433,7 +3433,7 @@ static int send_handle_init(int tot_thread_num)
 		if(g_ipv6_send_packet_enabled){
 			g_send_handle[i].raw_ipv6_fd = socket(PF_INET6, SOCK_RAW, IPPROTO_RAW);
 			if(g_send_handle[i].raw_ipv6_fd < 0){
-				sapp_log(30, ~0, ~0, "socket v6 error, %s\n", strerror(errno));
+				sapp_log(30, ~0, ~0, "create socket v6 error, %s\n", strerror(errno));
 				sapp_log(30, ~0, ~0, "Please disable IPv6 module, then try again!\n");
 				return -1;
 			}
@@ -3442,7 +3442,7 @@ static int send_handle_init(int tot_thread_num)
 		if(pcfg_io->inject_mode_single_gateway_device[0] != '\0'){
 			ret = g_send_handle[i].raw_eth_fd =  socket(PF_INET, SOCK_PACKET, htons(ETH_P_ALL));
 			if(g_send_handle[i].raw_eth_fd < 0){
-				sapp_log(30, ~0, ~0, "socket raw eth error, %s\n", strerror(errno));
+				sapp_log(30, ~0, ~0, "create socket raw eth error, %s\n", strerror(errno));
 				return -1;
 			}
 			ret = snprintf(g_send_handle[i].saddr_raw_eth.sa_data, sizeof(g_send_handle[i].saddr_raw_eth.sa_data), 
@@ -3467,6 +3467,14 @@ static int send_handle_init(int tot_thread_num)
 	return 0;
 }
 
+void send_handle_destroy(int thread_index)
+{
+	if(dl_io_fun_list.dl_io_free_send_handle){
+		dl_io_fun_list.dl_io_free_send_handle(thread_index, g_send_handle[thread_index].low_level_send_handle);
+	}
+}
+
+
 /* 2015-01-04 lijia add, ��pappƽ̨Ǩ�Ƶ�sappƽ̨, �ײ㷢���ӿڲ�ͬ */
 int MESA_sendpacket_ethlayer(int thread_num,const char *data, int data_len, unsigned int target_id)
 {
@@ -3507,6 +3515,28 @@ int sendpacket_init_new(int tot_thread_num)
 	return 0;
 }
 
+void sendpacket_destroy(int tot_thread_num)
+{
+	int i;
+
+	for(i = 0; i < tot_thread_num; i++){
+		if(g_send_handle[i].raw_ipv4_fd > 0){
+			close(g_send_handle[i].raw_ipv4_fd);
+		}
+		if(g_send_handle[i].raw_ipv6_fd > 0){
+			close(g_send_handle[i].raw_ipv6_fd);
+		}	
+		if(g_send_handle[i].raw_udp_fd > 0){
+			close(g_send_handle[i].raw_udp_fd);
+		}			
+		if(g_send_handle[i].raw_eth_fd > 0){
+			close(g_send_handle[i].raw_eth_fd);
+		}	
+
+		send_handle_destroy(i);
+	}
+}
+
 
 #define DIR_BIT_USE_MASK		(0x81)
 #define DIR_BIT_UNUSE_MASK		(0x7E)
diff --git a/src/packet_io/under_ddos.cpp b/src/packet_io/under_ddos.cpp
index 76255e7..83d56e1 100644
--- a/src/packet_io/under_ddos.cpp
+++ b/src/packet_io/under_ddos.cpp
@@ -268,6 +268,16 @@ int packet_io_under_ddos_init(void)
 	return 0;
 }
 
+void packet_io_under_ddos_destroy(void)
+{
+	if(0 == sapp_global_val->config.packet_io.under_ddos_config.enabled){
+		return;
+	}
+
+	cpu_limit_destroy(sapp_global_val->individual_fixed.under_ddos_handle);	
+	SAPP_FREE(sapp_global_val->individual_fixed.under_ddos_handle);
+}
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/sapp_dev/sapp_global_val.c b/src/sapp_dev/sapp_global_val.c
index 717bb6d..37013ba 100644
--- a/src/sapp_dev/sapp_global_val.c
+++ b/src/sapp_dev/sapp_global_val.c
@@ -215,6 +215,96 @@ void sapp_global_val_sanity_check(void)
 
 }
 
+void sapp_gval_init(void)
+{
+	int tseq;
+	
+	sapp_global_val = (sapp_global_t *)calloc(1, sizeof(sapp_global_t));
+	sapp_global_val->individual_volatile = (sapp_gval_individual_volatile_t *)calloc(1, sizeof(sapp_gval_individual_volatile_t));
+
+	for(tseq = 0; tseq < SAPP_MAX_THREADS; tseq++){
+		sapp_global_val->mthread_volatile[tseq] = (sapp_gval_mthread_t *)calloc(1, sizeof(sapp_gval_mthread_t));
+		sapp_global_val->individual_fixed.cpu_bind_core_id_per_thread[tseq] = -1; /* ��ʼ������Ϊ-1, ��ʾû�󶨵��κ�һ��cpu core */
+	}
+
+	/* set default value, maybe not input by user or config file */
+	snprintf(sapp_global_val->config.packet_io.pcap_dumpfile_name, NAME_MAX, "dumpfile");
+
+	ABBR_CURRENT_TIME = time(NULL);
+}
+
+void sapp_gval_destroy(void)
+{
+	int tseq;
+	
+	if(sapp_global_val->individual_volatile){
+		SAPP_FREE(sapp_global_val->individual_volatile);
+	}
+
+	for(tseq = 0; tseq < SAPP_MAX_THREADS; tseq++){
+		SAPP_FREE(sapp_global_val->mthread_volatile[tseq]);
+	}	
+
+	if(sapp_global_val->cla.cmd_args_array){
+		int i, args_num = sapp_global_val->cla.cmd_args_num;
+		sapp_cmd_args_val_t *free_opt;
+
+		for(i = 0; i < args_num; i++){
+			free_opt = sapp_global_val->cla.cmd_args_array[i];
+			SAPP_FREE(free_opt->long_opt_name);
+			SAPP_FREE(free_opt->arg_val.string_value);
+		}
+		
+		sapp_mem_free(sapp_global_val->cla.cmd_args_array);//�˴�ʹ����realloc׷��, Ҫ��sapp_mem_free()
+		sapp_global_val->cla.cmd_args_array = NULL;
+	}
+
+	sapp_config_file_link_t *pconfig = &sapp_global_val->config.cfg_file_path;
+	SAPP_FREE(pconfig->cfg_files_root_dir);
+	SAPP_FREE(pconfig->cfg_main_entry_relative);
+	SAPP_FREE(pconfig->cfg_main_entry_absolute);
+	SAPP_FREE(pconfig->cfg_sapp_log_relative);
+	SAPP_FREE(pconfig->cfg_sapp_log_absolute);
+	SAPP_FREE(pconfig->cfg_plug_list_relative);
+	SAPP_FREE(pconfig->cfg_plug_list_absolute);
+	SAPP_FREE(pconfig->cfg_project_list_relative);
+	SAPP_FREE(pconfig->cfg_project_list_absolute);
+	SAPP_FREE(pconfig->cfg_file_send_raw_pkt_relative);
+	SAPP_FREE(pconfig->cfg_file_send_raw_pkt_absolute);
+	SAPP_FREE(pconfig->cfg_file_vxlan_sport_map_relative);
+	SAPP_FREE(pconfig->cfg_file_vxlan_sport_map_absolute);
+	SAPP_FREE(pconfig->cfg_file_inline_dev_relative);
+	SAPP_FREE(pconfig->cfg_file_inline_dev_absolute);
+	SAPP_FREE(pconfig->cfg_file_necessary_plug_relative);
+	SAPP_FREE(pconfig->cfg_file_necessary_plug_absolute);
+	SAPP_FREE(pconfig->cfg_file_stream_compare_layer_relative);
+	SAPP_FREE(pconfig->cfg_file_stream_compare_layer_absolute);
+	SAPP_FREE(pconfig->cfg_file_vlan_flipping_relative);
+	SAPP_FREE(pconfig->cfg_file_vlan_flipping_absolute);
+	SAPP_FREE(pconfig->cfg_file_asymmetric_addr_layer_relative);
+	SAPP_FREE(pconfig->cfg_file_asymmetric_addr_layer_absolute);
+	SAPP_FREE(pconfig->cfg_file_well_known_port_relative);
+	SAPP_FREE(pconfig->cfg_file_well_known_port_absolute);
+
+	sapp_data_file_link_t *pdata = &sapp_global_val->config.data_file_path;
+	SAPP_FREE(pdata->data_files_root_dir);
+	SAPP_FREE(pdata->data_sapp_sysinfo_log_relative);
+	SAPP_FREE(pdata->data_sapp_sysinfo_log_absolute);
+	SAPP_FREE(pdata->data_sapp_fs2_log_relative);
+	SAPP_FREE(pdata->data_sapp_fs2_log_absolute);
+	SAPP_FREE(pdata->data_inline_keepalive_log_relative);
+	SAPP_FREE(pdata->data_inline_keepalive_log_absolute);
+	SAPP_FREE(pdata->data_load_plugin_stat_log_relative);
+	SAPP_FREE(pdata->data_load_plugin_stat_log_absolute);
+	SAPP_FREE(pdata->data_under_ddos_stat_log_relative);
+	SAPP_FREE(pdata->data_under_ddos_stat_log_absolute);
+
+	SAPP_FREE(sapp_global_val->config.library_file_path.libmarsio_file_path);
+
+	memset((void *)sapp_global_val, 0, sizeof(sapp_global_t));
+	SAPP_FREE(sapp_global_val);
+}
+
 
 #ifdef __cplusplus
 }
diff --git a/src/sapp_dev/sapp_init.c b/src/sapp_dev/sapp_init.c
index 0220e81..cca84b9 100644
--- a/src/sapp_dev/sapp_init.c
+++ b/src/sapp_dev/sapp_init.c
@@ -56,9 +56,10 @@ extern int sapp_assistant_init(void);
 extern char gdev_keepalive_ip_entry(const struct streaminfo *pstream,unsigned char routedir,int thread_seq, const struct mesa_ip4_hdr *ipv4_hdr);
 extern char gdev_keepalive_udp_entry(const struct streaminfo *a_udp,  void **pme, int thread_seq, const void *ip_hdr);
 
+static int _check_exit_cap_mode;
 static void forbid_call_exit_in_running_state(void)
 {
-	if(g_packet_io_cap_mode != CAP_MODEL_PCAP_DUMPFILE){
+	if(_check_exit_cap_mode != CAP_MODEL_PCAP_DUMPFILE){
 		printf("\033[41mCall exit() in running state is forbidden!!\033[0m\n");
 		abort();
 	}
@@ -346,7 +347,10 @@ void MESA_platform_run(void)
 	/* 
 		ʹ��atexit()ע��ĺ���, ������ע�ắ���෴, ���Ծ������Ӻ�atexit()ע��λ��,
 		�������г�ʼ�����֮��, packet_io_run()֮ǰ��
+
+		������sapp_gval_destroy()֮��, ȫ�ֱ���sapp_global_val����free��, �˴���ʱ��һ��.
 	*/
+	_check_exit_cap_mode = g_packet_io_cap_mode;
 	atexit(forbid_call_exit_in_running_state);
 
 	__sapp_timer_platform_run();
@@ -354,10 +358,10 @@ void MESA_platform_run(void)
 	packet_io_run();
 
 	sapp_set_current_state(SAPP_STATE_PROCESSING);
-	/* If TFE is run by systemd's notify, then tell the systemd our tfe is ready.
-	 * and disable the stderr log, only print logs into files */
-	if (check_is_started_by_notify())
-	{
+	/* 
+	   If sapp is started by systemd with notify, then tell the systemd sapp is ready.
+     */
+	if (check_is_started_by_notify()){
 		sd_notify(0, "READY=1");
 		sleep(1);
 	}
diff --git a/src/sapp_dev/sapp_plug.c b/src/sapp_dev/sapp_plug.c
index 50abf92..20df1ab 100644
--- a/src/sapp_dev/sapp_plug.c
+++ b/src/sapp_dev/sapp_plug.c
@@ -122,6 +122,37 @@ int libsapp_setup_env(int argc, char *argv[])
 }
 
 
+static void wait_for_all_io_threads(void)
+{
+	int tseq;
+
+	for(tseq = 0; tseq < g_packet_io_thread_num; tseq++){
+		pthread_join(sapp_global_val->individual_fixed.thread_obtain_id[tseq], NULL);
+	}
+
+	pthread_join(sapp_global_val->individual_fixed.thread_timer_event_id, NULL);
+}
+
+
+
+void libsapp_destroy_env_per_thread(int tseq)
+{
+	
+}
+
+
+
+
+void libsapp_destroy_env(void)
+{
+	sapp_set_current_state(SAPP_STATE_READY_TO_EXIT);
+
+	wait_for_all_io_threads();
+	
+	packet_io_exit();
+}
+
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/timer/sapp_timer.c b/src/timer/sapp_timer.c
index e4fa14c..96c140f 100644
--- a/src/timer/sapp_timer.c
+++ b/src/timer/sapp_timer.c
@@ -458,7 +458,7 @@ void *sapp_time_event_thread(void *arg)
 
 	last_log_time = time(NULL);
 	
-	while(1)
+	while(SAPP_STATE_PROCESSING == sapp_global_val->individual_volatile->current_state)
 	{
 		ABBR_CURRENT_TIME = time(NULL);
 		/*