TSG-7770, sapp支持按不同模块统计自身所占用的内存.

10 files changed, 109 insertions, 23 deletions

diff --git a/include/private/ip_reassembly.h b/include/private/ip_reassembly.h
index 783d82b..7df9276 100644
--- a/include/private/ip_reassembly.h
+++ b/include/private/ip_reassembly.h
@@ -114,8 +114,8 @@ void raw_frags_list_free_stream(int thread_num, struct streaminfo *stream);
 raw_ipfrag_list_t *frags_list_merge(raw_ipfrag_list_t *new_merge_list, raw_ipfrag_list_t *old_list);

 void __frags_list_check(raw_ipfrag_list_t *frags_list);

 int MESA_stream_list_cmp(struct streaminfo_private *stream1, struct streaminfo_private *stream2);

-void MESA_stream_list_free(struct streaminfo_private *heap_stream);

-struct streaminfo_private *MESA_stream_list_dup(struct streaminfo_private *stack_stream, int reverse);

+void MESA_stream_list_free(int mem_used_type, struct streaminfo_private *heap_stream);

+struct streaminfo_private *MESA_stream_list_dup(int mem_used_type, struct streaminfo_private *stack_stream, int reverse);

 

 #ifdef __cplusplus

 }

diff --git a/include/private/plugin.h b/include/private/plugin.h
index 7e9c2c3..9303e1a 100644
--- a/include/private/plugin.h
+++ b/include/private/plugin.h
@@ -4,6 +4,7 @@
 #include <netinet/ip.h>

 #include "stream.h"

 #include "sapp_limits.h"

+#include "sapp_mem.h"

 

 #ifdef __cplusplus

 extern "C" {

diff --git a/include/private/project_internal.h b/include/private/project_internal.h
index b31ac43..aeb63b8 100644
--- a/include/private/project_internal.h
+++ b/include/private/project_internal.h
@@ -8,8 +8,8 @@
 extern "C" {

 #endif

 

-void *project_requirement_create(int thread_num);

-void project_requirement_destroy(int thread_num, void *pproject);

+void *project_requirement_create(int mem_used_type, int thread_num);

+void project_requirement_destroy(int mem_used_type, int thread_num, void *pproject);

 

 raw_ipfrag_list_t *raw_ip_frag_list_move_pkt(struct streaminfo *stream);

 raw_ipfrag_list_t *raw_ip_frag_list_global_move_pkt(struct streaminfo *stream);

diff --git a/include/private/project_requirement.h b/include/private/project_requirement.h
index 7c0db8a..95d4fa5 100644
--- a/include/private/project_requirement.h
+++ b/include/private/project_requirement.h
@@ -14,8 +14,8 @@ extern "C" {
 #define RAW_FRAG_V6_MODULE_NAME		"ipv6_frag_list"

 

 

-void *project_requirement_create(int thread_num);

-void project_requirement_destroy(int thread_num, void *pproject);

+void *project_requirement_create(int mem_use_type, int thread_num);

+void project_requirement_destroy(int mem_used_type, int thread_num, void *pproject);

 int raw_ip_frag_list_stream_attach(struct streaminfo *stream);

 int raw_ip_frag_list_stream_detach(struct streaminfo *stream);

 void raw_frags_list_free_one(int thread_seq, void *project_req_value);

diff --git a/include/private/sapp_declaration.h b/include/private/sapp_declaration.h
index bf1ead4..a388b74 100644
--- a/include/private/sapp_declaration.h
+++ b/include/private/sapp_declaration.h
@@ -16,11 +16,6 @@ extern "C" {
 #define SAPP_MIN(a, b)       (((a) < (b)) ? (a) : (b))
 #endif
 
-#ifndef SAPP_FREE
-#define SAPP_FREE(m) do{if(m){free((void *)m); m = NULL;}}while(0)
-
-#endif
-
 
 /* ��Ƕ�ײ��̫��, ʹ�ü������ */
 #define ABBR_SAPP_LOG_LEVEL 	sapp_global_val->config.profiling.log.level
@@ -53,6 +48,8 @@ extern "C" {
 #define ABBR_INLINE_KEEPALIVE_LOG_DATA_FILE  sapp_global_val->config.data_file_path.data_inline_keepalive_log_absolute
 #define ABBR_LOAD_PLUG_STAT_LOG_DATA_FILE  sapp_global_val->config.data_file_path.data_load_plugin_stat_log_absolute
 #define ABBR_MARSIO_LIBRARY_FILE  	sapp_global_val->config.library_file_path.libmarsio_file_path
+#define ABBR_MEMORY_STAT_LOG_DATA_FILE  sapp_global_val->config.data_file_path.data_memory_stat_log_absolute
+
 
 
 extern sapp_global_t *sapp_global_val;
diff --git a/include/private/sapp_global_val.h b/include/private/sapp_global_val.h
index df3d6ab..7ca2c4a 100644
--- a/include/private/sapp_global_val.h
+++ b/include/private/sapp_global_val.h
@@ -5,6 +5,7 @@
 #include <netinet/in.h>
 #include "sapp_pkt_stat.h"
 #include "sapp_limits.h"
+#include "sapp_mem.h"
 #include "stream_inc/stream_base.h"
 
 enum pkt_dump_mode{
@@ -347,7 +348,10 @@ typedef struct{
 	const char *data_load_plugin_stat_log_absolute;		
 
 	const char *data_under_ddos_stat_log_relative;
-	const char *data_under_ddos_stat_log_absolute;		
+	const char *data_under_ddos_stat_log_absolute;	
+
+	const char *data_memory_stat_log_relative;
+	const char *data_memory_stat_log_absolute;		
 }sapp_data_file_link_t;
 
 typedef struct{
@@ -424,6 +428,11 @@ typedef struct{
 	long long time_cost_sum;
 }sapp_fuzzy_latency_stat_t;
 
+typedef struct{
+	long long mem_used_block[__SAPP_MEM_TYPE_MAX];
+	long long mem_used_bytes[__SAPP_MEM_TYPE_MAX];
+}sapp_mem_used_stat_t;
+
 struct __sapp_gval_mthread{
 	sapp_gval_mthread_sys_stat_t sys_stat;
 	unsigned short stream_seq_per_thread; /* ���ڼ���ȫ����ID */
@@ -432,6 +441,7 @@ struct __sapp_gval_mthread{
 	const raw_pkt_t *raw_pkt; /* ��ΪMESA_fakepacket_send_xxxϵ�к���û��raw_pkt����, Ҳû��stream_info, Ϊ����ǰ����, �����޸Ľӿ�, ����ȫ�ֱ������ڴ洢��ǰ�� */
 	void *dup_pkt_distinguish_handle; /* ����������ڿ���, inject, ipv4_tcp, ipv4_udp, ��ͬʹ��һ��bloom��� */
 	volatile int destory_env_done; //0:do nothing; 1:doing; 2:done
+	sapp_mem_used_stat_t mem_used_stat;
 }__attribute__ ((aligned (64))); /* for multi-thread, must 64byte alignment */
 typedef struct __sapp_gval_mthread sapp_gval_mthread_t; 
 
@@ -441,6 +451,7 @@ struct __sapp_gval_individual_volatile{
 	volatile long long current_time_ms;
 	volatile int current_state; /* refer to enum sapp_state_t  */
 	volatile int destory_env_done;
+	sapp_mem_used_stat_t mem_used_stat; /* ȫ�ֱ������ʼ���Ͳ�����ڴ� */
 }__attribute__ ((aligned (64)));
 typedef struct __sapp_gval_individual_volatile sapp_gval_individual_volatile_t;
 
diff --git a/include/private/sapp_mem.h b/include/private/sapp_mem.h
new file mode 100644
index 0000000..3f37a28
--- /dev/null
+++ b/include/private/sapp_mem.h
@@ -0,0 +1,75 @@
+#ifndef __SAPP_MEM_H_
+#define __SAPP_MEM_H_ 1
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define MEM_STAT_GLOBAL_THREAD_ID  -1  /* 全局变量内存的thread_id, 其实没有用, 保持接口一致, 用于非IO线程 */
+
+typedef enum{
+	__SAPP_MEM_TYPE_INIT = 0,
+
+	SAPP_MEM_STACK_LOCAL, /* 函数栈里临时申请的内存, 函数返回就会被释放 */
+	
+	SAPP_MEM_FIX_GLOBAL_VAL, 
+	SAPP_MEM_FIX_GLOBAL_STREAM, 
+	
+/*******************************************	
+	以上是全局变量, 或初始化后不变的内存
+*******************************************/
+	__SAPP_FIX_DYN_SEPARATOR,
+/*******************************************	
+	以下是多线程动态内存
+*******************************************/
+
+	SAPP_MEM_DYN_MEM_HDR, 		/* 为了增加这个统计, 增加的内存头部消耗 */
+	SAPP_MEM_DYN_IP_FRAG_PKT, /* 不太容易区别ipv4和ipv6, 此处统一计数 */
+	SAPP_MEM_DYN_TCP_STREAM,
+	SAPP_MEM_DYN_UDP_STREAM,
+	SAPP_MEM_DYN_TCP_HALF_STREAM, /* c2s, s2c半流结构 */
+	SAPP_MEM_DYN_UDP_HALF_STREAM,
+	SAPP_MEM_DYN_PLUG_CTRL, 	/* 表示插件管理必需的相关数据结构, 但不包括业务插件使用的内存 */
+	SAPP_MEM_DYN_TCP_UNORDER,
+	SAPP_MEM_DYN_TCP_SYN_OPT,	
+	SAPP_MEM_DYN_TCP_POLLING_RAW_PKT,
+	SAPP_MEM_DYN_UDP_POLLING_RAW_PKT,
+	SAPP_MEM_DYN_TCP_DETAIL,
+	SAPP_MEM_DYN_UDP_DETAIL,	
+	SAPP_MEM_DYN_TCP_FLOW_STAT,
+	SAPP_MEM_DYN_UDP_FLOW_STAT,
+	SAPP_MEM_DYN_TCP_PROJECT, /* 仅包括project必需的管理结构, 但不包括插件在project_add时申请的内存 */
+	SAPP_MEM_DYN_UDP_PROJECT,
+	SAPP_MEM_DYN_TCP_BRIDGE,
+	SAPP_MEM_DYN_UDP_BRIDGE,	
+	SAPP_MEM_DYN_PADDR, 		/* 每层地址所占的内存, 就不分具体协议了, 太麻烦了!! */
+
+	__SAPP_MEM_TYPE_MAX,
+}sapp_mem_type_t;
+
+void *sapp_mem_malloc(sapp_mem_type_t type, int thread_seq, int size);
+void *sapp_mem_calloc(sapp_mem_type_t type, int thread_seq, int size);
+void sapp_mem_free(sapp_mem_type_t type, int thread_seq, void *data);
+void *sapp_mem_realloc(sapp_mem_type_t type, int thread_seq, void *old_ptr, int size);
+void sapp_mem_stat_output(void);
+
+#ifndef SAPP_FREE
+#define SAPP_FREE(type,tid, mem) do{if(mem){sapp_mem_free(type, tid, (void *)mem); mem = NULL;}}while(0)
+#endif
+
+
+#ifndef SAPP_GLOBAL_ALLOC
+#define SAPP_GLOBAL_ALLOC(size) sapp_mem_calloc(SAPP_MEM_FIX_GLOBAL_VAL, MEM_STAT_GLOBAL_THREAD_ID, size)
+#endif
+
+#ifndef SAPP_GLOBAL_FREE
+#define SAPP_GLOBAL_FREE(mem) do{if(mem){sapp_mem_free(SAPP_MEM_FIX_GLOBAL_VAL, MEM_STAT_GLOBAL_THREAD_ID, (void *)mem); mem = NULL;}}while(0)
+#endif
+
+extern int g_sapp_mem_use_dictator;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/include/private/stream_internal.h b/include/private/stream_internal.h
index 6324a84..f970d7a 100644
--- a/include/private/stream_internal.h
+++ b/include/private/stream_internal.h
@@ -7,6 +7,7 @@
 #include "MESA_handle_logger.h"
 #include "mesa_pkt_dump.h"
 #include "sapp_limits.h"
+#include "sapp_mem.h"
 #include "MESA_jump_layer.h"
 
 //#define STREAM_BASE_MD5_CHECK		"dd09b3b11993cc835200db477dad7d4b"
@@ -360,7 +361,9 @@ struct sapp_global_mthread_t{
 }__attribute__ ((aligned (64)));
 
 typedef struct{
+#ifdef DEBUG
 	unsigned int magic; /* malloc�����0x4D4D4D4D,  ��MMMM ��16����; free�����0xFEFEFEFE */
+#endif	
 	unsigned int user_buf_size;
 }sapp_private_mem_t;
 
@@ -403,12 +406,12 @@ int gdev_block_init(void);
 int sapp_identify_broad_multicast_pkt(const void *this_layer_data, const raw_pkt_t *raw_pkt);
 int sapp_identify_tunnel_inner_pkt(const raw_pkt_t *raw_pkt);
 int sapp_assistant_init(void);
-int save_polling_inject_context(struct streaminfo_private *pstream_pr, const raw_pkt_t *raw_pkt);
-void free_polling_inject_context(struct streaminfo_private *pstream_pr);
-int update_polling_inject_context(struct streaminfo_private *pstream_pr, const raw_pkt_t *raw_pkt);
+int save_polling_inject_context(int mem_used_type, struct streaminfo_private *pstream_pr, const raw_pkt_t *raw_pkt);
+void free_polling_inject_context(int mem_used_type, struct streaminfo_private *pstream_pr);
+int update_polling_inject_context(int mem_used_type, struct streaminfo_private *pstream_pr, const raw_pkt_t *raw_pkt);
 unsigned long long get_global_stream_id(unsigned long long thread_seq);
 enum sapp_state_t sapp_get_current_state(void);
-extern struct streaminfo_private *copy_stream_info_to_heap(const struct streaminfo_private *stack_stream, int reverse);
+extern struct streaminfo_private *copy_stream_info_to_heap(int mem_used_type, const struct streaminfo_private *stack_stream, int reverse);
 enum addr_type_t addr_type_abbreviation_pton(const char *addr_type_abbreviation);
 const char *addr_type_abbreviation_ntop(enum addr_type_t addrtype);
 int sapp_identify_broad_multicast_init(void);
@@ -416,8 +419,8 @@ int sapp_identify_broad_multicast_init(void);
 int plugin_manage_init(void);
 void  *plugin_get_plug_entry(const char *plug_name, const char *plug_entry_type);
 int libsapp_setup_env(int argc, char *argv[]);
-void *stream_bridge_create_per_stream(int thread_num);
-void stream_bridge_destroy_per_stream(const struct streaminfo *stream, void *pbridge);
+void *stream_bridge_create_per_stream(int mem_used_type, int thread_num);
+void stream_bridge_destroy_per_stream(int mem_used_type, const struct streaminfo *stream, void *pbridge);
 char *sapp_strdup(const char *raw_s);
 int MESA_mkdir_p(const char *pathname, mode_t mode);
 int sapp_get_secondary_file_path(void);
@@ -430,14 +433,11 @@ void sapp_printf_colorful(int level, const char *format, ...);
 int set_transport_addr(struct streaminfo *this_stream, int enable_well_known_port, UINT16 sport, UINT16 dport);
 long long sapp_random(void);
 int sapp_usleep(int usec);
-
-void *sapp_mem_alloc(int size);
-void sapp_mem_free(void *data);
-void *sapp_mem_realloc(void *old_ptr, int size);
 void set_current_thread_cpu_affinity(int);
 int get_current_thread_cpu_affinity_id(void);
 void sapp_fs2_destroy(void);
 void sapp_gval_destroy(void);
+char *byte_convert_human(unsigned long long bytes, int interval, int multiple, char * byte_str);
 
 
 
diff --git a/include/private/stream_manage.h b/include/private/stream_manage.h
index 8c46bab..02dbc92 100644
--- a/include/private/stream_manage.h
+++ b/include/private/stream_manage.h
@@ -170,7 +170,7 @@ void update_opposite_addr_info(struct streaminfo_private *top_stream_pr, struct
 int polling_stream_timeout(int tid);

 int checkstreamorder(const struct streaminfo_private *top_stream_pr, struct streaminfo_private *pheap_stream_pr,

 		struct streaminfo_private *pstack_stream_pr, int *heap_streaminfo_skip_layer_num, int *stack_streaminfo_skip_layer_num);

-struct streaminfo_private *copy_stream_info_to_heap_single_layer(const struct streaminfo_private *stack_stream_pr, int reverse);

+struct streaminfo_private *copy_stream_info_to_heap_single_layer(int mem_used_type, const struct streaminfo_private *stack_stream_pr, int reverse);

 

 unsigned int sapp_jhash_4words(unsigned int a, unsigned int b, unsigned int c, unsigned int initval, unsigned int *out_val);

 

diff --git a/include/public/sapp_limits.h b/include/public/sapp_limits.h
index 07939d7..13fcd22 100644
--- a/include/public/sapp_limits.h
+++ b/include/public/sapp_limits.h
@@ -17,6 +17,8 @@
 
 #define SAPP_MAX_PLUG_ENTRY_NUM	(256) /* 所有插件的entry数量最大值 */
 
+#define STATS_STR_LEN	(32)
+
 
 #endif