diff options
| author | 郑超 <[email protected]> | 2018-11-27 14:38:14 +0800 |
|---|---|---|
| committer | 郑超 <[email protected]> | 2018-11-27 14:38:14 +0800 |
| commit | bcfb1e2ac886a2d1a0964079e49e6c73da778c02 (patch) | |
| tree | c9fabf25a5a6dbabd75dd6a8e0e731d45373dae4 | |
| parent | eddc68eaf099465a97a077ec954ab75fe058bead (diff) | |
| parent | a92967aa3fe6583865943574afab7d3f7c0be526 (diff) | |
Merge branch 'bugfix-memleakage-at-burn-feather' into 'master'v2.4.3
Bugfix memleakage at burn feather
See merge request MESA_framework/maat!15
| -rw-r--r-- | src/entry/Maat_api.cpp | 98 | ||||
| -rw-r--r-- | src/entry/Maat_command.cpp | 1 | ||||
| -rw-r--r-- | src/entry/Maat_rule.cpp | 80 | ||||
| -rw-r--r-- | src/entry/Maat_stat.cpp | 32 | ||||
| -rw-r--r-- | src/entry/UniversalBoolMatch.cpp | 1 | ||||
| -rw-r--r-- | src/inc_internal/Maat_rule_internal.h | 3 | ||||
| -rw-r--r-- | src/inc_internal/alignment_int64.h (renamed from src/inc_internal/aligment_int64.h) | 14 | ||||
| -rw-r--r-- | test/maat_demo.cpp | 2 | ||||
| -rw-r--r-- | test/test_maatframe.cpp | 23 |
9 files changed, 141 insertions, 113 deletions
diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index ca8a94c..8644ca5 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -10,7 +10,7 @@ #include "Maat_rule_internal.h" #include "Maat_utils.h" #include "dynamic_array.h" -#include "aligment_int64.h" +#include "alignment_int64.h" #include "config_monitor.h" #include "map_str2int.h" #include "rulescan.h" @@ -44,13 +44,13 @@ struct _Maat_table_info_t * acqurie_table(struct _Maat_feather_t* _feather,int t } inline void INC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num) { - aligment_int64_array_add(scanner->ref_cnt, thread_num, 1); + alignment_int64_array_add(scanner->ref_cnt, thread_num, 1); return; } inline void DEC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num) { - aligment_int64_array_add(scanner->ref_cnt, thread_num, -1); + alignment_int64_array_add(scanner->ref_cnt, thread_num, -1); return; } @@ -143,7 +143,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int { //short cut for rules contains one group scan_ret=shortcut_avilable_cnt; - aligment_int64_array_add(feather->orphan_group_saving, thread_num, 1); + alignment_int64_array_add(feather->orphan_group_saving, thread_num, 1); } else if(shortcut_avilable_cnt==0&®ion_hit_num==1&&_mid->hit_group_cnt==1&&is_last_region==1) { @@ -151,7 +151,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int //region_hit_num==1 : for current scan hitted rules, one and each other group may statisfy a compile rule. //_mid->hit_group_cnt==1: With pre scan hitted group rules, one group may staisfy a compile rule scan_ret=0; - aligment_int64_array_add(feather->last_region_saving, thread_num, 1); + alignment_int64_array_add(feather->last_region_saving, thread_num, 1); } else { @@ -187,7 +187,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int } if(result_cnt>0) { - aligment_int64_array_add(feather->hit_cnt,thread_num,1); + alignment_int64_array_add(feather->hit_cnt,thread_num,1); } return result_cnt; } @@ -380,7 +380,7 @@ struct _OUTER_scan_status_t* _make_outer_status(_Maat_feather_t *feather,int thr outer_mid->feather=feather; outer_mid->district_id=-1; outer_mid->thread_num=(unsigned short)thread_num; - aligment_int64_array_add(feather->outer_mid_cnt, thread_num,1); + alignment_int64_array_add(feather->outer_mid_cnt, thread_num,1); return outer_mid; } struct _OUTER_scan_status_t* grab_mid(scan_status_t* raw_mid,_Maat_feather_t* feather,int thread_num,int is_hit_region) @@ -400,7 +400,7 @@ struct _OUTER_scan_status_t* grab_mid(scan_status_t* raw_mid,_Maat_feather_t* fe if(_mid->inner==NULL) { _mid->inner=_make_inner_status(); - aligment_int64_array_add(feather->inner_mid_cnt,thread_num,1); + alignment_int64_array_add(feather->inner_mid_cnt,thread_num,1); } } return _mid; @@ -479,12 +479,12 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* feather->effect_interval_ms=60*1000; feather->scan_interval_ms=1*1000; feather->rule_scan_type=2; - feather->thread_call_cnt=aligment_int64_array_alloc(max_thread_num); - feather->outer_mid_cnt=aligment_int64_array_alloc(max_thread_num); - feather->inner_mid_cnt=aligment_int64_array_alloc(max_thread_num); - feather->hit_cnt=aligment_int64_array_alloc(max_thread_num); - feather->orphan_group_saving=aligment_int64_array_alloc(max_thread_num); - feather->last_region_saving=aligment_int64_array_alloc(max_thread_num); + feather->thread_call_cnt=alignment_int64_array_alloc(max_thread_num); + feather->outer_mid_cnt=alignment_int64_array_alloc(max_thread_num); + feather->inner_mid_cnt=alignment_int64_array_alloc(max_thread_num); + feather->hit_cnt=alignment_int64_array_alloc(max_thread_num); + feather->orphan_group_saving=alignment_int64_array_alloc(max_thread_num); + feather->last_region_saving=alignment_int64_array_alloc(max_thread_num); feather->maat_version=0; feather->last_full_version=0; feather->base_grp_seq=0; @@ -594,7 +594,6 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo break; case MAAT_OPT_STAT_ON: _feather->stat_on=1; - _feather->stat_handle=FS_create_handle(); break; case MAAT_OPT_PERF_ON: _feather->perf_on=1; @@ -794,7 +793,7 @@ int Maat_initiate_feather(Maat_feather_t feather) if(_feather->stat_on==1) { MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module , - "At initiation: MAAT_OPT_STAT_FILE_PATH not set,TURN OFF STAT trigger."); + "At initiation: MAAT_OPT_STAT_FILE_PATH not set, TURN OFF STAT trigger."); } _feather->stat_on=0; } @@ -810,11 +809,12 @@ int Maat_initiate_feather(Maat_feather_t feather) "Update with cumulative version OFF."); } - - maat_stat_init(_feather); + if(_feather->stat_on==1) + { + maat_stat_init(_feather); + } - pthread_t cfg_mon_t; - pthread_create(&cfg_mon_t, NULL, thread_rule_monitor, (void*)_feather); + pthread_create(&(_feather->cfg_mon_t), NULL, thread_rule_monitor, (void*)_feather); return 0; } @@ -889,6 +889,8 @@ void Maat_burn_feather(Maat_feather_t feather) { struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; _feather->still_working=0;//destroy will proceed in thread_rule_monitor + void* ret=NULL; + pthread_join(_feather->cfg_mon_t, &ret); return; } int Maat_table_register(Maat_feather_t feather,const char* table_name) @@ -1089,7 +1091,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id { sub_type=make_sub_type(table_id,charset,0); } - aligment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); + alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); scan_data_t scan_data; scan_data.text_data.text=data; scan_data.text_data.tlen=data_len; @@ -1136,7 +1138,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id } if(hit_region_cnt>0) { - aligment_int64_array_add(p_table->hit_cnt, thread_num,1); + alignment_int64_array_add(p_table->hit_cnt, thread_num,1); _mid=grab_mid(mid,_feather,thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, _mid->is_last_region, @@ -1228,7 +1230,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id { return 0; } - aligment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); + alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; @@ -1242,7 +1244,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id } else if(region_ret>0) { - aligment_int64_array_add(p_table->hit_cnt, thread_num,1); + alignment_int64_array_add(p_table->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, _mid->is_last_region, @@ -1323,7 +1325,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id { return 0; } - aligment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); + alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); ip_scan_data.rule_type=RULETYPE_IPv4; ip_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0); @@ -1364,7 +1366,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id } else if(region_ret>0) { - aligment_int64_array_add(p_table->hit_cnt, thread_num,1); + alignment_int64_array_add(p_table->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, @@ -1457,7 +1459,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, { sp->do_regex=1; } - aligment_int64_array_add(p_table->stream_num,thread_num,1); + alignment_int64_array_add(p_table->stream_num,thread_num,1); sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num); return sp; } @@ -1505,7 +1507,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para sp->feather->scan_err_cnt++; return -1; } - aligment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1); + alignment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1); region_result=scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*sp->thread_num; *detail_ret=0; if(sp->do_merge==1) @@ -1584,7 +1586,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para } if(hit_region_cnt>0) { - aligment_int64_array_add(p_table->hit_cnt, sp->thread_num,1); + alignment_int64_array_add(p_table->hit_cnt, sp->thread_num,1); _mid=grab_mid(mid, sp->feather,sp->thread_num, 1); compile_ret=region_compile(sp->feather,_mid->inner, _mid->is_last_region, @@ -1653,7 +1655,7 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para) struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _Maat_scanner_t* scanner=sp->feather->scanner; struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id]; - aligment_int64_array_add(p_table->stream_num,sp->thread_num,-1); + alignment_int64_array_add(p_table->stream_num,sp->thread_num,-1); if(sp->rs_stream_para!=NULL) { if(scanner!=NULL&&sp->version>=sp->feather->last_full_version) @@ -1697,12 +1699,6 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, _feather->scan_err_cnt++; return NULL; } - tmp_fuzzy_handle=SFH_instance(total_len); - if(tmp_fuzzy_handle==NULL) - { - _feather->scan_err_cnt++; - return NULL; - } struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1); scanner=_feather->scanner; sp->feather=_feather; @@ -1711,14 +1707,21 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, if(scanner==NULL) { return sp; + } + tmp_fuzzy_handle=SFH_instance(total_len); + if(tmp_fuzzy_handle==NULL) + { + _feather->scan_err_cnt++; + return NULL; } INC_SCANNER_REF(scanner, thread_num); + sp->table_id=table_id; sp->thread_num=thread_num; sp->total_len=total_len; sp->fuzzy_hash_handle=tmp_fuzzy_handle; pthread_mutex_init(&(sp->fuzzy_mutex),NULL); - aligment_int64_array_add(p_table->stream_num,thread_num,1); + alignment_int64_array_add(p_table->stream_num,thread_num,1); return sp; } @@ -1754,7 +1757,8 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int int do_query=0; GIE_result_t query_result[MAX_SCANNER_HIT_NUM]; int hit_region_cnt=0,compile_ret=0; - _compile_result_t compile_result[rule_num];//dynamic array + _compile_result_t compile_result[rule_num];//dynamic array + _Maat_table_info_t* p_table=NULL; if(data==NULL||data_len<=0) { return 0; @@ -1772,8 +1776,8 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { clock_gettime(CLOCK_MONOTONIC,&start); } - - aligment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1); + p_table=sp->feather->p_table_info[sp->table_id]; + alignment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1); pthread_mutex_lock(&(sp->fuzzy_mutex)); sp->process_offset+=SFH_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset); pthread_mutex_unlock(&(sp->fuzzy_mutex)); @@ -1808,8 +1812,8 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int goto fast_out; } if(hit_region_cnt>0) - { - sp->feather->p_table_info[sp->table_id]->hit_cnt++; + { + alignment_int64_array_add(p_table->hit_cnt, sp->thread_num, 1); _mid=grab_mid(mid,sp->feather, sp->thread_num,1); compile_ret=region_compile(sp->feather,_mid->inner, _mid->is_last_region, @@ -1844,7 +1848,7 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para) struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _Maat_scanner_t* scanner=sp->feather->scanner; struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id]; - aligment_int64_array_add(p_table->stream_num,sp->thread_num,-1); + alignment_int64_array_add(p_table->stream_num,sp->thread_num,-1); if(scanner!=NULL) { if(sp->version==sp->feather->maat_version) @@ -1967,7 +1971,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id { return 0; } - aligment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); + alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); INC_SCANNER_REF(my_scanner,thread_num); region_ret=GIE_query(gie_handle, data, data_len,region_result, MAX_SCANNER_HIT_NUM); @@ -1979,7 +1983,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id } else if(region_ret>0) { - aligment_int64_array_add(p_table->hit_cnt, thread_num,1); + alignment_int64_array_add(p_table->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, _mid->is_last_region, @@ -2019,12 +2023,12 @@ void Maat_clean_status(scan_status_t* mid) return; } _mid=(struct _OUTER_scan_status_t*)(*mid); - aligment_int64_array_add(_mid->feather->outer_mid_cnt,_mid->thread_num,-1); + alignment_int64_array_add(_mid->feather->outer_mid_cnt,_mid->thread_num,-1); if(_mid->inner!=NULL) { free(_mid->inner->hitted_group_id); free(_mid->inner); - aligment_int64_array_add(_mid->feather->inner_mid_cnt,_mid->thread_num,-1); + alignment_int64_array_add(_mid->feather->inner_mid_cnt,_mid->thread_num,-1); } _mid->feather=NULL; free(_mid); diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 0827b9d..740776e 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -2415,6 +2415,7 @@ int redis_flush_DB(redisContext* ctx, int db_index, void* logger) freeReplyObject(data_reply);
data_reply=_wrap_redisCommand(ctx,"MULTI");
+ freeReplyObject(data_reply);
redisAppendCommand(ctx,"FLUSHDB");
append_cmd_cnt++;
diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 146fe14..9def258 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -23,7 +23,7 @@ #include "json2iris.h" #include "cJSON.h" #include "dynamic_array.h" -#include "aligment_int64.h" +#include "alignment_int64.h" #include "config_monitor.h" #include "map_str2int.h" @@ -32,7 +32,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_4_20181120=1; +int MAAT_FRAME_VERSION_2_4_20181126=1; const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""}; @@ -334,7 +334,7 @@ int parse_accept_tag(const char* value, struct rule_tag** result, void* logger) } array=cJSON_GetObjectItem(json, "tags"); n_tags=cJSON_GetArraySize(array); - p=(struct rule_tag*)calloc(sizeof(struct rule_tag), n_tags); + p=ALLOC(struct rule_tag, n_tags); for(int i=0;i<n_tags;i++) { tag=cJSON_GetArrayItem(array, i); @@ -523,25 +523,25 @@ void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* s _Maat_table_info_t* create_table_info(int max_thread_num) { - struct _Maat_table_info_t*p=NULL; - p=(struct _Maat_table_info_t*)calloc(sizeof(struct _Maat_table_info_t),1); + struct _Maat_table_info_t*p=ALLOC(struct _Maat_table_info_t, 1); + p->conj_cnt=1; - p->scan_cnt=aligment_int64_array_alloc(max_thread_num); - p->scan_cpu_time=aligment_int64_array_alloc(max_thread_num); - p->input_bytes=aligment_int64_array_alloc(max_thread_num); - p->stream_num=aligment_int64_array_alloc(max_thread_num); - p->hit_cnt=aligment_int64_array_alloc(max_thread_num); + p->scan_cnt=alignment_int64_array_alloc(max_thread_num); + p->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); + p->input_bytes=alignment_int64_array_alloc(max_thread_num); + p->stream_num=alignment_int64_array_alloc(max_thread_num); + p->hit_cnt=alignment_int64_array_alloc(max_thread_num); p->cross_cache_size=0; p->quick_expr_switch=0; return p; } void destroy_table_info(struct _Maat_table_info_t*p) { - aligment_int64_array_free(p->scan_cnt); - aligment_int64_array_free(p->scan_cpu_time); - aligment_int64_array_free(p->input_bytes); - aligment_int64_array_free(p->stream_num); - aligment_int64_array_free(p->hit_cnt); + alignment_int64_array_free(p->scan_cnt); + alignment_int64_array_free(p->scan_cpu_time); + alignment_int64_array_free(p->input_bytes); + alignment_int64_array_free(p->stream_num); + alignment_int64_array_free(p->hit_cnt); if(p->cb_info!=NULL) { dynamic_array_destroy(p->cb_info->cache_lines, free); @@ -688,7 +688,7 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* FILE*fp=NULL; char line[MAX_TABLE_LINE_SIZE]; int i=0,ret=0,table_cnt=0; - char table_type_str[16],not_care[1024], tmp_str[32]; + char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0}; MESA_htable_handle string2int_map=map_create(); struct _Maat_table_info_t*p=NULL; struct _Maat_table_info_t*conj_table=NULL; @@ -996,18 +996,19 @@ void _destroy_compile_rule(struct _Maat_compile_inner_t * compile_rule) pthread_rwlock_wrlock(&(compile_rule->rwlock)); + if(db_compile_rule!=NULL) { + for(i=0; i<table->ex_data_num; i++) + { + rule_ex_data_free(&(db_compile_rule->m_rule_head), db_compile_rule->service_defined, compile_rule->ads+i, table->ex_desc+i); + compile_rule->ads[i]=NULL; + } + free(db_compile_rule->service_defined); free(db_compile_rule); compile_rule->db_c_rule=NULL; - } - for(i=0; table!=NULL && i<table->ex_data_num; i++) - { - rule_ex_data_free(&(db_compile_rule->m_rule_head), db_compile_rule->service_defined, compile_rule->ads+i, table->ex_desc+i); - compile_rule->ads[i]=NULL; - } - + } free(compile_rule->ads); pthread_rwlock_unlock(&(compile_rule->rwlock)); @@ -1230,7 +1231,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t scanner->dedup_expr_num=0; scanner->max_thread_num=scan_thread_num; //optimized for CPU cache_alignment 64 - scanner->ref_cnt=aligment_int64_array_alloc(scan_thread_num); + scanner->ref_cnt=alignment_int64_array_alloc(scan_thread_num); scanner->region_update_q=MESA_lqueue_create(0,0); scanner->region=rulescan_initialize(scan_thread_num); @@ -2396,7 +2397,7 @@ void compatible_group_udpate(struct _Maat_table_info_t* table,int region_id,int } void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) { - struct db_str_rule_t* maat_str_rule=(struct db_str_rule_t*)malloc(sizeof(struct db_str_rule_t)); + struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1); int ret=0,db_hexbin=0,rule_type=0; switch(table->table_type) { @@ -3033,7 +3034,7 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger) destroy_group_rule(bag->group_rule); break; case GARBAGE_SCANNER: - ref_cnt=aligment_int64_array_sum(bag->scanner->ref_cnt,bag->scanner->max_thread_num); + ref_cnt=alignment_int64_array_sum(bag->scanner->ref_cnt,bag->scanner->max_thread_num); if(ref_cnt==0) { MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module, @@ -3513,7 +3514,7 @@ void *thread_rule_monitor(void *arg) old_scanner->version, feather->scanner->version); } assert(old_scanner->tomb_ref==feather->garbage_q); - feather->zombie_rs_stream+=aligment_int64_array_sum(old_scanner->ref_cnt,old_scanner->max_thread_num); + feather->zombie_rs_stream+=alignment_int64_array_sum(old_scanner->ref_cnt,old_scanner->max_thread_num); garbage_bagging(GARBAGE_SCANNER, old_scanner, feather->garbage_q); } feather->update_tmp_scanner=NULL; @@ -3548,9 +3549,8 @@ void *thread_rule_monitor(void *arg) MESA_htable_destroy(feather->map_tablename2id,free); destroy_maat_scanner(feather->scanner); garbage_bury(feather->garbage_q,0,feather->logger); + assert(0==MESA_lqueue_get_count(feather->garbage_q)); MESA_lqueue_destroy(feather->garbage_q,lqueue_destroy_cb,NULL); - FS_stop(&(feather->stat_handle)); - int i=0; for(i=0;i<MAX_TABLE_NUM;i++) { @@ -3561,12 +3561,12 @@ void *thread_rule_monitor(void *arg) destroy_table_info(feather->p_table_info[i]); feather->p_table_info[i]=NULL; } - aligment_int64_array_free(feather->thread_call_cnt); - aligment_int64_array_free(feather->inner_mid_cnt); - aligment_int64_array_free(feather->outer_mid_cnt); - aligment_int64_array_free(feather->hit_cnt); - aligment_int64_array_free(feather->orphan_group_saving); - aligment_int64_array_free(feather->last_region_saving); + alignment_int64_array_free(feather->thread_call_cnt); + alignment_int64_array_free(feather->inner_mid_cnt); + alignment_int64_array_free(feather->outer_mid_cnt); + alignment_int64_array_free(feather->hit_cnt); + alignment_int64_array_free(feather->orphan_group_saving); + alignment_int64_array_free(feather->last_region_saving); if(feather->REDIS_MODE_ON==1&&feather->redis_read_ctx!=NULL) { pthread_mutex_lock(&(feather->redis_write_lock)); @@ -3576,6 +3576,16 @@ void *thread_rule_monitor(void *arg) feather->redis_write_ctx=NULL; pthread_mutex_unlock(&(feather->redis_write_lock)); } + for(i=0; i<feather->n_tags; i++) + { + free(feather->accept_tags[i].tag_name); + free(feather->accept_tags[i].tag_val); + } + free(feather->accept_tags); + if(feather->stat_on&& feather->stat_handle) + { + FS_stop(&(feather->stat_handle)); + } free(feather); return NULL; } diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp index 6658989..ed7190b 100644 --- a/src/entry/Maat_stat.cpp +++ b/src/entry/Maat_stat.cpp @@ -1,5 +1,5 @@ #include "Maat_rule_internal.h" -#include "aligment_int64.h" +#include "alignment_int64.h" #include <time.h> #include <MESA/field_stat.h> enum MAAT_FS_STATUS{ @@ -51,6 +51,8 @@ void maat_stat_init(struct _Maat_feather_t* feather) FS_set_para(feather->stat_handle, PRINT_MODE, &value, sizeof(value)); value=0; FS_set_para(feather->stat_handle, CREATE_THREAD, &value, sizeof(value)); + + FS_set_para(feather->stat_handle, APP_NAME, feather->instance_name, strlen(feather->instance_name)+1); feather->fs_status_id[STATUS_VERSION]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"version"); feather->fs_status_id[STATUS_THRED_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"thread"); @@ -149,11 +151,11 @@ void maat_stat_init(struct _Maat_feather_t* feather) } void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num) { - aligment_int64_array_add(p_table->scan_cnt,thread_num,1); - aligment_int64_array_add(p_table->input_bytes,thread_num,scan_len); + alignment_int64_array_add(p_table->scan_cnt,thread_num,1); + alignment_int64_array_add(p_table->input_bytes,thread_num,scan_len); if(start!=NULL&&end!=NULL) { - aligment_int64_array_add(p_table->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec); + alignment_int64_array_add(p_table->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec); } return; } @@ -171,11 +173,11 @@ void maat_stat_output(struct _Maat_feather_t* feather) time_t now; struct _Maat_table_info_t* p_table=NULL; time(&now); - active_thread_num=aligment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num); - outer_mid_cnt=aligment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num); - inner_mid_cnt=aligment_int64_array_sum(feather->inner_mid_cnt,feather->scan_thread_num); - orphan_group_saving=aligment_int64_array_sum(feather->orphan_group_saving,feather->scan_thread_num); - last_region_saving=aligment_int64_array_sum(feather->last_region_saving,feather->scan_thread_num); + active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num); + outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num); + inner_mid_cnt=alignment_int64_array_sum(feather->inner_mid_cnt,feather->scan_thread_num); + orphan_group_saving=alignment_int64_array_sum(feather->orphan_group_saving,feather->scan_thread_num); + last_region_saving=alignment_int64_array_sum(feather->last_region_saving,feather->scan_thread_num); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_THRED_NUM], 0,FS_OP_SET,active_thread_num); @@ -231,7 +233,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) p_table->regex_rule_cnt); total_regex_num+= p_table->regex_rule_cnt; - table_stream_num=aligment_int64_array_sum(p_table->stream_num,feather->scan_thread_num); + table_stream_num=alignment_int64_array_sum(p_table->stream_num,feather->scan_thread_num); FS_operate(feather->stat_handle, p_table->stat_line_id, feather->fs_column_id[COLUMN_TABLE_STREAM_NUM], @@ -239,7 +241,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) table_stream_num); total_stream_cnt+= table_stream_num; - table_scan_cnt=aligment_int64_array_sum(p_table->scan_cnt,feather->scan_thread_num); + table_scan_cnt=alignment_int64_array_sum(p_table->scan_cnt,feather->scan_thread_num); FS_operate(feather->stat_handle, p_table->stat_line_id, feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], @@ -247,7 +249,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) table_scan_cnt); total_scan_cnt+=table_scan_cnt; - table_input_bytes=aligment_int64_array_sum(p_table->input_bytes,feather->scan_thread_num); + table_input_bytes=alignment_int64_array_sum(p_table->input_bytes,feather->scan_thread_num); FS_operate(feather->stat_handle, p_table->stat_line_id, feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], @@ -256,7 +258,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) total_input_bytes+=table_input_bytes; if(feather->perf_on==1) { - table_scan_cpu_time=aligment_int64_array_sum(p_table->scan_cpu_time,feather->scan_thread_num); + table_scan_cpu_time=alignment_int64_array_sum(p_table->scan_cpu_time,feather->scan_thread_num); table_scan_cpu_time/=1000; FS_operate(feather->stat_handle, p_table->stat_line_id, @@ -266,7 +268,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) total_cpu_time+=table_scan_cpu_time; } - table_hit_cnt=aligment_int64_array_sum(p_table->hit_cnt,feather->scan_thread_num); + table_hit_cnt=alignment_int64_array_sum(p_table->hit_cnt,feather->scan_thread_num); FS_operate(feather->stat_handle, p_table->stat_line_id, feather->fs_column_id[COLUMN_TABLE_HIT_CNT], @@ -312,7 +314,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) FS_OP_SET, total_cpu_time); } - total_hit_cnt=aligment_int64_array_sum(feather->hit_cnt,feather->scan_thread_num); + total_hit_cnt=alignment_int64_array_sum(feather->hit_cnt,feather->scan_thread_num); FS_operate(feather->stat_handle, feather->total_stat_id, feather->fs_column_id[COLUMN_TABLE_HIT_CNT], diff --git a/src/entry/UniversalBoolMatch.cpp b/src/entry/UniversalBoolMatch.cpp index 477cc6d..ff91d60 100644 --- a/src/entry/UniversalBoolMatch.cpp +++ b/src/entry/UniversalBoolMatch.cpp @@ -252,6 +252,7 @@ void boolexpr_destroy(void * instance) for(unsigned int i=0; i<matcher->max_thread_num; i++) { delete [] matcher->thread_data[i].bitmap; + delete [] matcher->thread_data[i].matched_bitmap; } delete [] matcher->thread_data; delete matcher; diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index b94273d..c266116 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -14,7 +14,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -#include "aligment_int64.h" +#include "alignment_int64.h" #include <pthread.h> #include <iconv.h> @@ -415,6 +415,7 @@ struct _Maat_feather_t char group_tn[MAX_TABLE_NAME_LEN]; pthread_mutex_t backgroud_update_mutex; unsigned char decrypt_key[MAX_TABLE_NAME_LEN]; + pthread_t cfg_mon_t; char redis_ip[MAX_TABLE_NAME_LEN]; int redis_port; diff --git a/src/inc_internal/aligment_int64.h b/src/inc_internal/alignment_int64.h index 1ae4200..a342205 100644 --- a/src/inc_internal/aligment_int64.h +++ b/src/inc_internal/alignment_int64.h @@ -1,18 +1,18 @@ -#ifndef H_ALIGMENT_INT64_H_INCLUDE -#define H_ALIGMENT_INT64_H_INCLUDE +#ifndef H_ALIGNMENT_INT64_H_INCLUDE +#define H_ALIGNMENT_INT64_H_INCLUDE #include <stdlib.h> #define CPU_CACHE_ALIGMENT 64 typedef long long* mcore_long_t; -inline long long *aligment_int64_array_alloc(int size) +inline mcore_long_t alignment_int64_array_alloc(int size) { long long *ret=NULL; ret=(long long*)calloc(CPU_CACHE_ALIGMENT,size); return ret; } -inline long long aligment_int64_array_sum(mcore_long_t array,int size) +inline long long alignment_int64_array_sum(mcore_long_t array,int size) { long long sum=0; int offset=0,i=0; @@ -23,13 +23,13 @@ inline long long aligment_int64_array_sum(mcore_long_t array,int size) } return sum; } -inline long long aligment_int64_array_add(mcore_long_t array,int offset,long long op_val) +inline long long alignment_int64_array_add(mcore_long_t array,int offset,long long op_val) { int idx=(CPU_CACHE_ALIGMENT/sizeof(long long))*offset; array[idx]+=op_val; return array[idx]; } -inline long long aligment_int64_array_cnt(mcore_long_t array,int size) +inline long long alignment_int64_array_cnt(mcore_long_t array,int size) { int offset=0,i=0; int cnt=0; @@ -43,7 +43,7 @@ inline long long aligment_int64_array_cnt(mcore_long_t array,int size) } return cnt; } -inline void aligment_int64_array_free(mcore_long_t array) +inline void alignment_int64_array_free(mcore_long_t array) { free(array); } diff --git a/test/maat_demo.cpp b/test/maat_demo.cpp index 864db40..b536059 100644 --- a/test/maat_demo.cpp +++ b/test/maat_demo.cpp @@ -323,7 +323,7 @@ int test_digest_scan(Maat_feather_t feather,const char* table_name,const char* f { printf("fopen %s error.\n",file_name); } - Maat_stream_scan_string_end(&sp); + Maat_stream_scan_digest_end(&sp); return ret; } int test_plugin_table(Maat_feather_t feather,const char* table_name, diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 58c1f16..e2d36f5 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -235,7 +235,7 @@ TEST(DigestScan, Pure) } } fclose(fp); - Maat_stream_scan_string_end(&sp); + Maat_stream_scan_digest_end(&sp); EXPECT_GE(hit_cnt, 1); Maat_clean_status(&mid); return; @@ -743,10 +743,8 @@ class MaatFileTest : public testing::Test { protected: - static void SetUpTestCase() { - void *logger=NULL; const char* rule_folder="./ntcrule/full/index"; logger=MESA_create_runtime_log_handle("test_maat_file.log",0); const char* table_info="./t2_tableinfo.conf"; @@ -766,12 +764,16 @@ protected: static void TearDownTestCase() { Maat_burn_feather(_shared_feather_f); + MESA_destroy_runtime_log_handle(logger); } // Some expensive resource shared by all tests. - static Maat_feather_t _shared_feather_f; + static Maat_feather_t _shared_feather_f; + static void *logger; }; Maat_feather_t MaatFileTest::_shared_feather_f; +void* MaatFileTest::logger; + TEST_F(MaatFileTest, StreamFiles) { #define StreamScan_StreamFiles @@ -851,7 +853,6 @@ protected: static void SetUpTestCase() { - void *logger=NULL; logger=MESA_create_runtime_log_handle("test_maat_redis.log",0); _shared_feather=Maat_feather(g_iThreadNum, table_info_path, logger); @@ -874,13 +875,17 @@ protected: static void TearDownTestCase() { Maat_burn_feather(_shared_feather); + MESA_destroy_runtime_log_handle(logger); } // Some expensive resource shared by all tests. - static Maat_feather_t _shared_feather; + static Maat_feather_t _shared_feather; + static void *logger; static int linger_timeout; }; Maat_feather_t MaatCmdTest::_shared_feather; +void* MaatCmdTest::logger; + int MaatCmdTest::linger_timeout; int test_add_expr_command(Maat_feather_t feather,const char* region_table,int config_id, int timeout,int label_id, const char* keywords) { @@ -1107,6 +1112,7 @@ TEST_F(MaatCmdTest, SetIP) int table_id=0; struct Maat_rule_t result; + memset(&result, 0, sizeof(result)); scan_status_t mid=NULL; table_id=Maat_table_register(feather,region_table); ASSERT_GE(table_id, 0); @@ -1377,6 +1383,9 @@ int main(int argc, char ** argv) Maat_initiate_feather(g_feather); printf("Maat initiating, see %s\n",log_file); - return RUN_ALL_TESTS(); + int ret=RUN_ALL_TESTS(); + Maat_burn_feather(g_feather); + MESA_destroy_runtime_log_handle(g_logger); + return ret; } |
