diff options
| author | liuwentan <[email protected]> | 2023-09-22 14:59:44 +0800 |
|---|---|---|
| committer | liuwentan <[email protected]> | 2023-09-22 14:59:44 +0800 |
| commit | d55ca3595a8bdc9281a765a5b39b49135e7e415d (patch) | |
| tree | d4040ba3ee389dbd5312043257935d242e139c8a | |
| parent | 1e6c87c3d43cd7cb549bf021f0aef404a384add0 (diff) | |
[FEATURE]add hit_path&hit_group enable APIv4.0.42
| -rw-r--r-- | include/maat.h | 24 | ||||
| -rw-r--r-- | src/inc_internal/maat_rule.h | 2 | ||||
| -rw-r--r-- | src/maat_api.c | 25 | ||||
| -rw-r--r-- | src/maat_compile.c | 135 | ||||
| -rw-r--r-- | test/maat_framework_gtest.cpp | 84 |
5 files changed, 122 insertions, 148 deletions
diff --git a/include/maat.h b/include/maat.h index 5f08bfd..689af33 100644 --- a/include/maat.h +++ b/include/maat.h @@ -55,16 +55,16 @@ enum maat_update_type { MAAT_UPDATE_TYPE_INC }; -enum maat_list_type { - MAAT_LIST_TYPE_FULL = 1, - MAAT_LIST_TYPE_INC -}; - enum maat_expr_engine { MAAT_EXPR_ENGINE_HS = 0, //default engine(hyperscan) MAAT_EXPR_ENGINE_RS //rulescan }; +enum maat_list_type { + MAAT_LIST_TYPE_FULL = 1, + MAAT_LIST_TYPE_INC +}; + struct ip_addr { int ip_type; //4: IPv4, 6: IPv6 union { @@ -152,6 +152,10 @@ int maat_options_set_stat_file(struct maat_options *opts, const char *stat_filen int maat_options_set_expr_engine(struct maat_options *opts, enum maat_expr_engine engine); +int maat_options_set_hit_path_enabled(struct maat_options *opts); + +int maat_options_set_hit_group_enabled(struct maat_options *opts); + /* maat_instance API */ struct maat *maat_new(struct maat_options *opts, const char *table_info_path); void maat_free(struct maat *instance); @@ -290,18 +294,16 @@ int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *pat size_t maat_state_get_scan_count(struct maat_state *state); /** - * @brief get direct hit groups(full or incremental) + * @brief direct group means group corresponding to item * - * @param type: - * MAAT_LIST_TYPE_FULL => get all hit groups after maat_state_new - * MAAT_LIST_TYPE_INC => get hit groups for this scan * NOTE: hit groups may be duplicated */ int maat_state_get_direct_hit_groups(struct maat_state *state, enum maat_list_type type, - struct maat_hit_group *group_array, size_t array_size); + struct maat_hit_group *group_array, + size_t array_size); /** - * @brief get indirect hit groups + * @brief indirect group means superior group * * NOTE: hit groups may be duplicated */ diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index 252cdfe..8996ebc 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -119,6 +119,8 @@ struct maat_options { int stat_on; int perf_on; + int hit_path_on; + int hit_group_on; int deferred_load_on; int maat_json_is_gzipped; diff --git a/src/maat_api.c b/src/maat_api.c index dd24fbc..d4962d2 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -268,6 +268,28 @@ int maat_options_set_expr_engine(struct maat_options *opts, return 0; } +int maat_options_set_hit_path_enabled(struct maat_options *opts) +{ + if (NULL == opts) { + return -1; + } + + opts->hit_path_on = 1; + + return 0; +} + +int maat_options_set_hit_group_enabled(struct maat_options *opts) +{ + if (NULL == opts) { + return -1; + } + + opts->hit_group_on = 1; + + return 0; +} + int maat_options_set_logger(struct maat_options *opts, const char *log_path, enum log_level level) { @@ -1956,7 +1978,8 @@ size_t maat_state_get_scan_count(struct maat_state *state) } int maat_state_get_direct_hit_groups(struct maat_state *state, enum maat_list_type type, - struct maat_hit_group *group_array, size_t array_size) + struct maat_hit_group *group_array, + size_t array_size) { if (NULL == state || NULL == group_array || 0 == array_size) { return -1; diff --git a/src/maat_compile.c b/src/maat_compile.c index b56d391..53339c1 100644 --- a/src/maat_compile.c +++ b/src/maat_compile.c @@ -144,15 +144,15 @@ struct maat_compile_state { time_t compile_rt_version; UT_array *internal_hit_paths; - UT_array *internal_inc_hit_paths; UT_array *all_hit_clauses; UT_array *this_scan_hit_clauses; + UT_array *direct_hit_groups; UT_array *indirect_hit_groups; }; UT_icd ut_literal_id_icd = {sizeof(struct maat_literal_id), NULL, NULL, NULL}; UT_icd ut_clause_id_icd = {sizeof(long long), NULL, NULL, NULL}; -UT_icd ut_indirect_hit_group_icd = {sizeof(struct maat_hit_group), NULL, NULL, NULL}; +UT_icd ut_maat_hit_group_icd = {sizeof(struct maat_hit_group), NULL, NULL, NULL}; UT_icd ut_hit_path_icd = {sizeof(struct maat_internal_hit_path), NULL, NULL, NULL}; static struct maat_compile *maat_compile_new(long long compile_id) @@ -1346,10 +1346,10 @@ struct maat_compile_state *maat_compile_state_new(void) struct maat_compile_state *compile_state = ALLOC(struct maat_compile_state, 1); utarray_new(compile_state->internal_hit_paths, &ut_hit_path_icd); - utarray_new(compile_state->internal_inc_hit_paths, &ut_hit_path_icd); utarray_new(compile_state->all_hit_clauses, &ut_clause_id_icd); utarray_new(compile_state->this_scan_hit_clauses, &ut_clause_id_icd); - utarray_new(compile_state->indirect_hit_groups, &ut_indirect_hit_group_icd); + utarray_new(compile_state->direct_hit_groups, &ut_maat_hit_group_icd); + utarray_new(compile_state->indirect_hit_groups, &ut_maat_hit_group_icd); return compile_state; } @@ -1366,9 +1366,9 @@ void maat_compile_state_reset(struct maat_compile_state *compile_state) compile_state->not_clause_hit_flag = 0; utarray_clear(compile_state->internal_hit_paths); - utarray_clear(compile_state->internal_inc_hit_paths); utarray_clear(compile_state->all_hit_clauses); utarray_clear(compile_state->this_scan_hit_clauses); + utarray_clear(compile_state->direct_hit_groups); utarray_clear(compile_state->indirect_hit_groups); } @@ -1387,13 +1387,6 @@ void maat_compile_state_free(struct maat_compile_state *compile_state, compile_state->internal_hit_paths = NULL; } - if (compile_state->internal_inc_hit_paths != NULL) { - free_bytes += utarray_size(compile_state->internal_inc_hit_paths) * - sizeof(struct maat_internal_hit_path); - utarray_free(compile_state->internal_inc_hit_paths); - compile_state->internal_inc_hit_paths = NULL; - } - if (compile_state->all_hit_clauses != NULL) { free_bytes += utarray_size(compile_state->all_hit_clauses) * sizeof(long long); utarray_free(compile_state->all_hit_clauses); @@ -1406,6 +1399,12 @@ void maat_compile_state_free(struct maat_compile_state *compile_state, compile_state->this_scan_hit_clauses = NULL; } + if (compile_state->direct_hit_groups != NULL) { + free_bytes += utarray_size(compile_state->direct_hit_groups) * sizeof(struct maat_hit_group); + utarray_free(compile_state->direct_hit_groups); + compile_state->direct_hit_groups = NULL; + } + if (compile_state->indirect_hit_groups != NULL) { free_bytes += utarray_size(compile_state->indirect_hit_groups) * sizeof(struct maat_hit_group); utarray_free(compile_state->indirect_hit_groups); @@ -1531,40 +1530,52 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr return (n_hit_path + new_hit_path_cnt); } -static void maat_compile_state_update_indirect_group(struct maat_compile_state *compile_state, - long long *group_ids, size_t n_group_id, - int vtable_id) +static void maat_compile_state_update_direct_hit_groups(UT_array *hit_group_array, + struct maat_item *hit_items, + size_t n_hit_items, int vtable_id) { - if (NULL == compile_state) { + if (NULL == hit_group_array) { return; } struct maat_hit_group hit_group; - for (size_t i = 0; i < n_group_id; i++) { - hit_group.item_id = 0; - hit_group.group_id = group_ids[i]; + for (size_t i = 0; i < n_hit_items; i++) { + hit_group.item_id = hit_items[i].item_id; + hit_group.group_id = hit_items[i].group_id; hit_group.vtable_id = vtable_id; - utarray_push_back(compile_state->indirect_hit_groups, &hit_group); + utarray_push_back(hit_group_array, &hit_group); } } -static void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, - long long item_id, long long group_id, - int vtable_id, int Nth_scan) +static void maat_compile_state_update_indirect_hit_groups(UT_array *hit_group_array, + long long *group_ids, + size_t n_group_ids, int vtable_id) { - if (compile_state->Nth_scan != Nth_scan) { - assert(compile_state->this_scan_hit_item_flag == 0); - compile_state->Nth_scan = Nth_scan; - utarray_clear(compile_state->this_scan_hit_clauses); - } + if (NULL == hit_group_array) { + return; + } - maat_compile_hit_path_add(compile_state->internal_inc_hit_paths, item_id, - group_id, vtable_id, Nth_scan); + struct maat_hit_group hit_group; + for (size_t i = 0; i < n_group_ids; i++) { + hit_group.item_id = 0; + hit_group.group_id = group_ids[i]; + hit_group.vtable_id = vtable_id; + utarray_push_back(hit_group_array, &hit_group); + } +} - maat_compile_hit_path_add(compile_state->internal_hit_paths, item_id, - group_id, vtable_id, Nth_scan); +static void maat_compile_state_update_hit_paths(struct maat_compile_state *compile_state, + struct maat_item *hit_items, size_t n_hit_items, + int vtable_id, int Nth_scan) +{ + if (NULL == compile_state) { + return; + } - compile_state->this_scan_hit_item_flag = 1; + for (size_t i = 0; i < n_hit_items; i++) { + maat_compile_hit_path_add(compile_state->internal_hit_paths, hit_items[i].item_id, + hit_items[i].group_id, vtable_id, Nth_scan); + } } static void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, @@ -2077,24 +2088,38 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items, state->thread_id, 1); } - int g2g_table_id = table_manager_get_group2group_table_id(maat_inst->tbl_mgr); - void *g2g_rt = table_manager_get_runtime(maat_inst->tbl_mgr, g2g_table_id); + struct maat_compile_state *compile_state = state->compile_state; + if (hit_cnt > 0) { + if (compile_state->Nth_scan != state->scan_cnt) { + assert(compile_state->this_scan_hit_item_flag == 0); + compile_state->Nth_scan = state->scan_cnt; + utarray_clear(compile_state->this_scan_hit_clauses); + } + compile_state->this_scan_hit_item_flag = 1; + } for (i = 0; i < hit_cnt; i++) { hit_group_ids[i] = hit_items[i].group_id; } + int g2g_table_id = table_manager_get_group2group_table_id(maat_inst->tbl_mgr); + void *g2g_rt = table_manager_get_runtime(maat_inst->tbl_mgr, g2g_table_id); + long long super_group_ids[MAX_SCANNER_HIT_GROUP_NUM]; size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_rt, hit_group_ids, hit_cnt, super_group_ids, MAX_SCANNER_HIT_GROUP_NUM); - for (i = 0; i < hit_cnt; i++) { - maat_compile_state_update_hit_path(state->compile_state, hit_items[i].item_id, - hit_items[i].group_id, vtable_id, state->scan_cnt); - } + if (1 == maat_inst->opts.hit_path_on) { + maat_compile_state_update_hit_paths(compile_state, hit_items, hit_cnt, + vtable_id, state->scan_cnt); + } - maat_compile_state_update_indirect_group(state->compile_state, super_group_ids, - super_group_cnt, vtable_id); + if (1 == maat_inst->opts.hit_group_on) { + maat_compile_state_update_direct_hit_groups(compile_state->direct_hit_groups, + hit_items, hit_cnt, vtable_id); + maat_compile_state_update_indirect_hit_groups(compile_state->indirect_hit_groups, + super_group_ids, super_group_cnt, vtable_id); + } /* update hit clause */ int compile_table_id = table_manager_get_default_compile_table_id(maat_inst->tbl_mgr); @@ -2109,12 +2134,12 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items, } for (int j = 0; j < super_group_cnt; j++) { - maat_compile_state_update_hit_clause(state->compile_state, compile_rt, + maat_compile_state_update_hit_clause(compile_state, compile_rt, super_group_ids[j], vtable_id); } for (int j = 0; j < hit_cnt; j++) { - maat_compile_state_update_hit_clause(state->compile_state, compile_rt, + maat_compile_state_update_hit_clause(compile_state, compile_rt, hit_group_ids[j], vtable_id); } } @@ -2148,27 +2173,19 @@ size_t maat_compile_state_get_direct_hit_groups(struct maat_compile_state *compi if (NULL == compile_state) { return 0; } - - UT_array *wanted_hit_path = NULL; - if (type == MAAT_LIST_TYPE_FULL) { - wanted_hit_path = compile_state->internal_hit_paths; - } else if (type == MAAT_LIST_TYPE_INC) { - wanted_hit_path = compile_state->internal_inc_hit_paths; - } + UT_array *direct_hit_group = compile_state->direct_hit_groups; size_t i = 0; - struct maat_internal_hit_path *path = NULL; - for (i = 0; i < utarray_len(wanted_hit_path) && i < array_size; i++) { - path = (struct maat_internal_hit_path *)utarray_eltptr(wanted_hit_path, i); - group_array[i].item_id = path->item_id; - group_array[i].group_id = path->group_id; - group_array[i].vtable_id = path->vtable_id; + struct maat_hit_group *group = NULL; + for (i = 0; i < utarray_len(direct_hit_group) && i < array_size; i++) { + group = (struct maat_hit_group *)utarray_eltptr(direct_hit_group, i); + group_array[i].item_id = group->item_id; + group_array[i].group_id = group->group_id; + group_array[i].vtable_id = group->vtable_id; } - if (type == MAAT_LIST_TYPE_INC) { - utarray_clear(compile_state->internal_inc_hit_paths); - } + utarray_clear(compile_state->direct_hit_groups); return i; } diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 6451d32..4eb779d 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -518,6 +518,7 @@ protected: maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); maat_options_set_accept_tags(opts, accept_tags); + maat_options_set_hit_path_enabled(opts); _shared_maat_inst = maat_new(opts, table_info_path); maat_options_free(opts); @@ -767,6 +768,7 @@ protected: maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); maat_options_set_accept_tags(opts, accept_tags); + maat_options_set_hit_path_enabled(opts); //maat_options_set_expr_engine(opts, MAAT_EXPR_ENGINE_HS); //default _shared_maat_inst = maat_new(opts, table_info_path); @@ -1437,6 +1439,7 @@ protected: maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); maat_options_set_accept_tags(opts, accept_tags); maat_options_set_expr_engine(opts, MAAT_EXPR_ENGINE_RS); + maat_options_set_hit_path_enabled(opts); _shared_maat_inst = maat_new(opts, table_info_path); maat_options_free(opts); @@ -4072,6 +4075,7 @@ protected: maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); maat_options_set_accept_tags(opts, accept_tags); + maat_options_set_hit_path_enabled(opts); _shared_maat_inst = maat_new(opts, table_info_path); maat_options_free(opts); @@ -4884,6 +4888,8 @@ protected: maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); + maat_options_set_hit_path_enabled(opts); + maat_options_set_hit_group_enabled(opts); _shared_maat_inst = maat_new(opts, table_info_path); assert(_shared_maat_inst != NULL); @@ -6443,14 +6449,7 @@ TEST_F(MaatCmdTest, HitGroup) { struct maat_hit_group hit_groups[128]; memset(hit_groups, 0, sizeof(hit_groups)); - int n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128); - EXPECT_EQ(n_hit_group, 1); - EXPECT_EQ(hit_groups[0].item_id, item1_id); - EXPECT_EQ(hit_groups[0].group_id, group1_id); - EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); - - memset(hit_groups, 0, sizeof(hit_groups)); - n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128); + int n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128); EXPECT_EQ(n_hit_group, 1); EXPECT_EQ(hit_groups[0].item_id, item1_id); EXPECT_EQ(hit_groups[0].group_id, group1_id); @@ -6477,17 +6476,6 @@ TEST_F(MaatCmdTest, HitGroup) { scan_count = maat_state_get_scan_count(state); EXPECT_EQ(scan_count, 2); - n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128); - EXPECT_EQ(n_hit_group, 2); - - EXPECT_EQ(hit_groups[0].item_id, item1_id); - EXPECT_EQ(hit_groups[0].group_id, group1_id); - EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); - - EXPECT_EQ(hit_groups[1].item_id, item2_id); - EXPECT_EQ(hit_groups[1].group_id, group2_id); - EXPECT_EQ(hit_groups[1].vtable_id, http_res_table_id); - memset(hit_groups, 0, sizeof(hit_groups)); n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128); EXPECT_EQ(n_hit_group, 1); @@ -6516,21 +6504,6 @@ TEST_F(MaatCmdTest, HitGroup) { scan_count = maat_state_get_scan_count(state); EXPECT_EQ(scan_count, 3); - n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128); - EXPECT_EQ(n_hit_group, 3); - - EXPECT_EQ(hit_groups[0].item_id, item1_id); - EXPECT_EQ(hit_groups[0].group_id, group1_id); - EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); - - EXPECT_EQ(hit_groups[1].item_id, item2_id); - EXPECT_EQ(hit_groups[1].group_id, group2_id); - EXPECT_EQ(hit_groups[1].vtable_id, http_res_table_id); - - EXPECT_EQ(hit_groups[2].item_id, item4_id); - EXPECT_EQ(hit_groups[2].group_id, group4_id); - EXPECT_EQ(hit_groups[2].vtable_id, 0); //physical table(keywords_table) vtable_id is 0 - int ip_table_id = maat_get_table_id(maat_inst, ip_table_name); ASSERT_GT(ip_table_id, 0); @@ -6543,25 +6516,6 @@ TEST_F(MaatCmdTest, HitGroup) { scan_count = maat_state_get_scan_count(state); EXPECT_EQ(scan_count, 4); - n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128); - EXPECT_EQ(n_hit_group, 4); - - EXPECT_EQ(hit_groups[0].item_id, item1_id); - EXPECT_EQ(hit_groups[0].group_id, group1_id); - EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); - - EXPECT_EQ(hit_groups[1].item_id, item2_id); - EXPECT_EQ(hit_groups[1].group_id, group2_id); - EXPECT_EQ(hit_groups[1].vtable_id, http_res_table_id); - - EXPECT_EQ(hit_groups[2].item_id, item4_id); - EXPECT_EQ(hit_groups[2].group_id, group4_id); - EXPECT_EQ(hit_groups[2].vtable_id, 0); //physical table(ip_table) vtable_id is 0 - - EXPECT_EQ(hit_groups[3].item_id, item3_id); - EXPECT_EQ(hit_groups[3].group_id, group3_id); - EXPECT_EQ(hit_groups[3].vtable_id, 0); //physical table(keywords_table) vtable_id is 0 - memset(hit_groups, 0, sizeof(hit_groups)); n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128); EXPECT_EQ(n_hit_group, 2); @@ -6582,30 +6536,6 @@ TEST_F(MaatCmdTest, HitGroup) { EXPECT_EQ(scan_count, 5); memset(hit_groups, 0, sizeof(hit_groups)); - n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128); - EXPECT_EQ(n_hit_group, 5); - - EXPECT_EQ(hit_groups[0].item_id, item1_id); - EXPECT_EQ(hit_groups[0].group_id, group1_id); - EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); - - EXPECT_EQ(hit_groups[1].item_id, item2_id); - EXPECT_EQ(hit_groups[1].group_id, group2_id); - EXPECT_EQ(hit_groups[1].vtable_id, http_res_table_id); - - EXPECT_EQ(hit_groups[2].item_id, item4_id); - EXPECT_EQ(hit_groups[2].group_id, group4_id); - EXPECT_EQ(hit_groups[2].vtable_id, 0); //physical table(ip_table) vtable_id is 0 - - EXPECT_EQ(hit_groups[3].item_id, item3_id); - EXPECT_EQ(hit_groups[3].group_id, group3_id); - EXPECT_EQ(hit_groups[3].vtable_id, 0); //physical table(keywords_table) vtable_id is 0 - - EXPECT_EQ(hit_groups[4].item_id, item5_id); - EXPECT_EQ(hit_groups[4].group_id, group1_id); - EXPECT_EQ(hit_groups[4].vtable_id, 0); - - memset(hit_groups, 0, sizeof(hit_groups)); n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128); EXPECT_EQ(n_hit_group, 1); |
