🐞 fix(sapp watchdog): 修复优雅退出过程中，可能触发watchdog超时的bugv4.2.80

6 files changed, 71 insertions, 11 deletions

diff --git a/include/public/stream_inc/stream_control.h b/include/public/stream_inc/stream_control.h
index ebfc860..76c7b4a 100644
--- a/include/public/stream_inc/stream_control.h
+++ b/include/public/stream_inc/stream_control.h
@@ -204,6 +204,7 @@ enum sapp_state_t{
 	SAPP_STATE_PKT_IO_INITED,		

 	SAPP_STATE_PROCESSING,      /* 8 */

 	SAPP_STATE_READY_TO_EXIT,   /* 9,  pcap dumpfile mode, or recv custom signal */

+	SAPP_STATE_EXIT,

 };

 

 

diff --git a/src/packet_io/packet_io.c b/src/packet_io/packet_io.c
index 5b42bdf..2baf917 100644
--- a/src/packet_io/packet_io.c
+++ b/src/packet_io/packet_io.c
@@ -1414,39 +1414,64 @@ void packet_io_exit(void)
 		SAPP_GLOBAL_FREE(g_send_buf_pool[i]);
 	}	
 
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call global_sapp_timer_destroy()!");
 	global_sapp_timer_destroy();
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call gdev_keepalive_plug_destroy()!");
 	gdev_keepalive_plug_destroy();	
 	if(sapp_global_val->config.packet_io.destroy_all_plug_enabled == 1)
 	{
+		sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call plugctrl_destroy_all_plug()!");
 		plugctrl_destroy_all_plug();
 	}
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call sendpacket_destroy()!");
 	sendpacket_destroy(g_packet_io_thread_num);
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call packet_io_lib_destroy()!");
 	packet_io_lib_destroy();
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call packet_io_device_alias_destroy()!");
 	packet_io_device_alias_destroy();
 
-	sapp_log_handle_destroy();
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call sapp_fs2_destroy()!");
 	sapp_fs2_destroy();
+
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call packet_io_under_ddos_destroy()!");
 	packet_io_under_ddos_destroy();
 
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call ipv4_frag_exit()!");
 	ipv4_frag_exit();
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call ipv6_frag_exit()!");
 	ipv6_frag_exit();
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call plugin_manage_destroy()!");
 	plugin_manage_destroy();
 
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call global_project_req_destroy()!");
 	global_project_req_destroy();
 
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call stream_bridge_destroy()!");
 	stream_bridge_destroy();
 
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call cycle_pkt_dump_destroy()!");
 	cycle_pkt_dump_destroy();
 
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call stream_destroy_plugin()!");
 	stream_destroy_plugin();
 
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call global_stream_destroy()!");
 	global_stream_destroy();
 
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call sapp_dup_pkt_destroy()!");
 	sapp_dup_pkt_destroy();
 
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call sapp_breakpad_destroy()!");
 	sapp_breakpad_destroy();
 
+	sapp_runtime_log(RLOG_LV_INFO, "packet_io_exit, call sapp_log_handle_destroy()!");
+	sapp_log_handle_destroy();
+
+	printf("packet_io_exit excuting, set SAPP_STATE_EXIT!");	
+	sapp_set_current_state(SAPP_STATE_EXIT);
+	usleep(100);
 	/****** ע��:�ͷ�ȫ�ֱ���Ҫ�������һ��!! ******/
+	printf("packet_io_exit, call sapp_gval_destroy\n");
 	sapp_gval_destroy();
 
 	return;
diff --git a/src/packet_io/packet_io_pcap.c b/src/packet_io/packet_io_pcap.c
index f142bca..bb20857 100644
--- a/src/packet_io/packet_io_pcap.c
+++ b/src/packet_io/packet_io_pcap.c
@@ -1442,6 +1442,7 @@ thread_exit:
 	sapp_usleep(1000);

 	sapp_set_current_state(SAPP_STATE_READY_TO_EXIT);

 	if(exit_prog_flag){

+		printf("pcap_io_thread SAPP_STATE_READY_TO_EXIT, call exit() !\n");

 		exit(0);

 	}

 	

diff --git a/src/plugin/src/plugin.c b/src/plugin/src/plugin.c
index 5044f1f..19cea4d 100644
--- a/src/plugin/src/plugin.c
+++ b/src/plugin/src/plugin.c
@@ -1963,12 +1963,14 @@ void plugctrl_destroy_all_plug()
 	stProtocolPlugInfo *pro_plug_info, *pro_plug_next;
 	int i;
 
+	sapp_runtime_log(RLOG_LV_INFO, "plugctrl_destroy_all_plug, begin destory all plugin !");
 	/***** call all plugins destory callback fun *****/
 	plug_info = g_platform_plug_info;
 	while (plug_info)
 	{
 		if (plug_info->fun_destroy)
 		{
+			sapp_runtime_log(RLOG_LV_INFO, "plugctrl_destroy_all_plug, platform, call [%s] destroy", plug_info->plugname);
 			plug_info->fun_destroy();
 		}
 		plug_info = plug_info->next;
@@ -1979,6 +1981,7 @@ void plugctrl_destroy_all_plug()
 	{
 		if (pro_plug_info->fun_destroy)
 		{
+			sapp_runtime_log(RLOG_LV_INFO, "plugctrl_destroy_all_plug, protocol, call [%s] destroy", pro_plug_info->plugname);
 			pro_plug_info->fun_destroy();
 		}
 		pro_plug_info = pro_plug_info->next;
@@ -1989,13 +1992,17 @@ void plugctrl_destroy_all_plug()
 	{
 		if (plug_info->fun_destroy)
 		{
+			sapp_runtime_log(RLOG_LV_INFO, "plugctrl_destroy_all_plug, business, call [%s] destroy", plug_info->plugname);
 			plug_info->fun_destroy();
 		}
 		plug_info = plug_info->next;
 	}
+
+	sapp_runtime_log(RLOG_LV_INFO, "plugctrl_destroy_all_plug, begin dlcose all plugin !");
 	/***** dlclose all plugins library handle  *****/
 	plug_info = g_business_plug_info;
 	while(plug_info){
+		sapp_runtime_log(RLOG_LV_INFO, "plugctrl_destroy_all_plug, business, dlcose [%s]", plug_info->plugname);
 		dlclose(plug_info->filepoint);
 		plug_next = plug_info->next;
 		SAPP_GLOBAL_FREE(plug_info);
@@ -2005,6 +2012,7 @@ void plugctrl_destroy_all_plug()
 	
 	pro_plug_info = g_protocol_plug_info;
 	while(pro_plug_info){
+		sapp_runtime_log(RLOG_LV_INFO, "plugctrl_destroy_all_plug, protocol, dlcose [%s]", pro_plug_info->plugname);
 		dlclose(pro_plug_info->filepoint);
 		pro_plug_next = pro_plug_info->next;
 		SAPP_GLOBAL_FREE(pro_plug_info);
@@ -2014,6 +2022,7 @@ void plugctrl_destroy_all_plug()
 	
 	plug_info = g_platform_plug_info;
 	while(plug_info){
+		sapp_runtime_log(RLOG_LV_INFO, "plugctrl_destroy_all_plug, platform, dlcose [%s]", plug_info->plugname);
 		dlclose(plug_info->filepoint);
 		plug_next = plug_info->next;
 		SAPP_GLOBAL_FREE(plug_info);
@@ -2028,6 +2037,7 @@ void plugctrl_destroy_all_plug()
 	
 	MESA_destroy_runtime_log_handle(g_plugin_runtime_handle);
 	g_plugin_runtime_handle = NULL;
+	sapp_runtime_log(RLOG_LV_INFO, "plugctrl_destroy_all_plug, finish !");
 
 }
 
diff --git a/src/sapp_dev/sapp_plug.c b/src/sapp_dev/sapp_plug.c
index c66dabc..6dc20ce 100644
--- a/src/sapp_dev/sapp_plug.c
+++ b/src/sapp_dev/sapp_plug.c
@@ -45,7 +45,7 @@ static void signal_hup_handler(int signo)
 
 static void signal_term_handler(int signo)
 {
-	printf("SIGTERM recviced!\n");
+	printf("signal_term_handler, SIGTERM recviced!\n");
 	sapp_global_val->individual_volatile->recv_signal_SIGTERM = 1;
 }
 
@@ -117,6 +117,8 @@ static void sapp_graceful_exit(void)
 		return;
 	}
 
+	sapp_runtime_log(RLOG_LV_FATAL, "graceful_exit begin!");
+
 	if((DEPOLYMENT_MODE_INLINE == sapp_global_val->config.packet_io.depolyment_mode_bin)
 	 && (OVERLAY_MODE_VXLAN == sapp_global_val->config.packet_io.packet_io_tunnel.overlay_mode_bin)){
 		SAPP_TLV_T stop_keepalive_opt = {};
@@ -124,7 +126,7 @@ static void sapp_graceful_exit(void)
 		stop_keepalive_opt.length = sizeof(int);
 		stop_keepalive_opt.int_value = 0;
 		gdev_keepalive_set_opt(&stop_keepalive_opt);
-		sapp_runtime_log(RLOG_LV_FATAL, "recv SIGTERM, for inline and vxlan mode, stop keepalive and waiting for all packet have been processed!\n");
+		sapp_runtime_log(RLOG_LV_FATAL, "graceful_exit, for inline and vxlan mode, stop keepalive and waiting for all packet have been processed!");
 		
 		/* �ر��˱���, ���м�ʮ�������ʱ, inline�豸�Ż�������Ϊ���ܶ˲�������, 
 		   ������̽�������, �ܶ����ᶪ��,
@@ -133,8 +135,9 @@ static void sapp_graceful_exit(void)
 		while(sapp_calc_line_pps() != 0);
 	}
 	
-	sapp_runtime_log(RLOG_LV_FATAL, "recv SIGTERM, graceful exit!\n");
+	sapp_runtime_log(RLOG_LV_FATAL, "graceful_exit clear keepalive done, begin libsapp_destroy_env!");
 	libsapp_destroy_env();
+	printf("sapp_graceful_exit end, call exit() !\n");
 	_exit(0);
 }
 
@@ -146,6 +149,7 @@ static void *sapp_monitor_thread(void *args)
 		usleep(10000);
 	}
 
+	sapp_runtime_log(RLOG_LV_FATAL, "sapp_monitor_thread  begin libsapp_destroy_env!\n");
 	libsapp_destroy_env();
 
 	return NULL;
@@ -264,7 +268,7 @@ void libsapp_destroy_env(void)
 		�������ͬ������, ��double free��!!
 		�˴������ü�������һ���Ⱥ��߼�˳��.
 	*/
-	
+	sapp_runtime_log(RLOG_LV_FATAL, "libsapp_destroy_env begin, try to sync exit state !");	
 	if(MESA_ATOMIC_INC(g_sapp_destory_env_running_state) != 1){ 
 		/* ��ʼ��ʱ��0, INC����1, ˵����������һ���߳�������destroy����, �����ȴ�, ֱ������Ϊֹ */
 		MESA_ATOMIC_DEC(g_sapp_destory_env_running_state);
@@ -284,9 +288,15 @@ void libsapp_destroy_env(void)
 		exit_process = 1;
 	}
 
+	sapp_runtime_log(RLOG_LV_INFO, "libsapp_destroy_env excuting, set SAPP_STATE_READY_TO_EXIT !");	
 	sapp_set_current_state(SAPP_STATE_READY_TO_EXIT);
+
+	sapp_runtime_log(RLOG_LV_INFO, "libsapp_destroy_env excuting, call packet_io_stop() !");	
 	packet_io_stop();
+	
+	sapp_runtime_log(RLOG_LV_INFO, "libsapp_destroy_env excuting, call wait_for_all_io_threads() !");	
 	wait_for_all_io_threads();
+	sapp_runtime_log(RLOG_LV_INFO, "libsapp_destroy_env excuting, call packet_io_exit() !");	
 	packet_io_exit();
 
 	/* ������Դ��free��, ��ȥ���ü��� */
@@ -294,6 +304,7 @@ void libsapp_destroy_env(void)
 
 	MESA_ATOMIC_SET(g_sapp_destory_env_done_state, SAPP_DESTROY_DONE_FLAG);	
 	if(exit_process){
+		printf("libsapp_destroy_env end, call exit() !\n");	
 		exit(0);
 	}	
 }
diff --git a/src/timer/sapp_timer.c b/src/timer/sapp_timer.c
index 33fef27..bed4443 100644
--- a/src/timer/sapp_timer.c
+++ b/src/timer/sapp_timer.c
@@ -442,6 +442,7 @@ void *sapp_time_event_thread(void *arg)
 
 	last_log_time = time(NULL);
 
+	sapp_runtime_log(RLOG_LV_INFO, "sapp_time_event_thread, init!");
 	/* Ϊ�˱�����docker, �����, �ڴ治��������, �˴������� current_state != SAPP_STATE_PROCESSING �ж����� 
        �������ֹ�dumpfileģʽ����ȡһ��pcap��, ˲����˳���, ���´˴���ѭ��������!!
 	*/
@@ -449,12 +450,9 @@ void *sapp_time_event_thread(void *arg)
 		sapp_usleep(100);
 	}
 
+
 	update_packet_io_status_sw = 1;
-	if(SAPP_STATE_READY_TO_EXIT == sapp_global_val->individual_volatile->current_state)
-	{
-		sd_notify(0, "WATCHDOG_USEC=600000000");
-		return;
-	}
+	sapp_runtime_log(RLOG_LV_INFO, "sapp_time_event_thread, SAPP_STATE_PROCESSING state, loop start!");
 	while(SAPP_STATE_PROCESSING == sapp_global_val->individual_volatile->current_state)
 	{
 		ABBR_CURRENT_TIME = time(NULL);
@@ -478,7 +476,21 @@ void *sapp_time_event_thread(void *arg)
 		
 		sapp_usleep(10);
 	}
-	
+	if(sapp_global_val->individual_volatile->current_state >= SAPP_STATE_READY_TO_EXIT)
+	{
+		sapp_runtime_log(RLOG_LV_INFO, "sapp_time_event_thread, SAPP_STATE_READY_TO_EXIT state, keep sd_notify watchdog=1!");
+		while(sd_watchdog_enabled(0, NULL))
+		{
+			if (sapp_global_val->individual_volatile->current_state >= SAPP_STATE_EXIT)
+			{
+				sd_notify(0, "STOPPING=1");
+				printf("sapp_time_event_thread, state not SAPP_STATE_PROCESSING, loop end, thread exit!\n");
+				break;
+			}
+			sd_notify(0, "WATCHDOG=1");
+			usleep(10);
+		}
+	}
 	return NULL;
 }