diff options
| author | liuwentan <[email protected]> | 2023-10-13 17:55:51 +0800 |
|---|---|---|
| committer | liuwentan <[email protected]> | 2023-10-13 17:55:51 +0800 |
| commit | 2252578b23f1edf3248d39177256f75908755853 (patch) | |
| tree | 3ffc4047564ca42fff0731dff792bd8cdaf26918 | |
| parent | f6a396a691dcd880737dd6d41924a3dc250ef76b (diff) | |
[FEATURE]support xx_plugin statistics(new API maat_register_thread)v4.0.48
| -rw-r--r-- | include/maat.h | 6 | ||||
| -rw-r--r-- | src/inc_internal/maat_bool_plugin.h | 4 | ||||
| -rw-r--r-- | src/inc_internal/maat_fqdn_plugin.h | 4 | ||||
| -rw-r--r-- | src/inc_internal/maat_ip_plugin.h | 4 | ||||
| -rw-r--r-- | src/inc_internal/maat_ipport_plugin.h | 4 | ||||
| -rw-r--r-- | src/inc_internal/maat_plugin.h | 4 | ||||
| -rw-r--r-- | src/inc_internal/maat_rule.h | 1 | ||||
| -rw-r--r-- | src/maat_api.c | 62 | ||||
| -rw-r--r-- | src/maat_bool_plugin.c | 31 | ||||
| -rw-r--r-- | src/maat_fqdn_plugin.c | 31 | ||||
| -rw-r--r-- | src/maat_ip_plugin.c | 33 | ||||
| -rw-r--r-- | src/maat_ipport_plugin.c | 33 | ||||
| -rw-r--r-- | src/maat_plugin.c | 34 | ||||
| -rw-r--r-- | src/maat_stat.c | 8 | ||||
| -rw-r--r-- | src/maat_table.c | 5 | ||||
| -rw-r--r-- | test/maat_framework_perf_gtest.cpp | 617 | ||||
| -rw-r--r-- | test/maat_json.json | 128 |
17 files changed, 775 insertions, 234 deletions
diff --git a/include/maat.h b/include/maat.h index 76bdbf0..76e9497 100644 --- a/include/maat.h +++ b/include/maat.h @@ -160,6 +160,12 @@ int maat_options_set_hit_group_enabled(struct maat_options *opts); struct maat *maat_new(struct maat_options *opts, const char *table_info_path); void maat_free(struct maat *instance); +/** + * Each thread can call this function initially, maat will maintain the thread_id internally, + * So it's no need to pass thread_id by maat_scan_xx and xx_plugin_get_ex_data API +*/ +void maat_register_thread(struct maat *instance); + /* maat helper API */ int maat_helper_read_column(const char *table_line, int Nth_column, size_t *column_offset, size_t *column_len); diff --git a/src/inc_internal/maat_bool_plugin.h b/src/inc_internal/maat_bool_plugin.h index c8bd280..3cc4f86 100644 --- a/src/inc_internal/maat_bool_plugin.h +++ b/src/inc_internal/maat_bool_plugin.h @@ -57,6 +57,10 @@ int bool_plugin_runtime_get_ex_data(void *bool_plugin_runtime, unsigned long lon long long bool_plugin_runtime_update_err_count(void *bool_plugin_runtime); +void bool_plugin_runtime_scan_inc(void *bool_plugin_runtime, int thread_id); + +long long bool_plugin_runtime_scan_count(void *bool_plugin_runtime); + #ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_fqdn_plugin.h b/src/inc_internal/maat_fqdn_plugin.h index 94aa670..2fc2f3b 100644 --- a/src/inc_internal/maat_fqdn_plugin.h +++ b/src/inc_internal/maat_fqdn_plugin.h @@ -61,6 +61,10 @@ void fqdn_rule_free(struct FQDN_rule *fqdn_rule); long long fqdn_plugin_runtime_update_err_count(void *fqdn_plugin_runtime); +void fqdn_plugin_runtime_scan_inc(void *fqdn_plugin_runtime, int thread_id); + +long long fqdn_plugin_runtime_scan_count(void *fqdn_plugin_runtime); + #ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_ip_plugin.h b/src/inc_internal/maat_ip_plugin.h index 759d859..3e65355 100644 --- a/src/inc_internal/maat_ip_plugin.h +++ b/src/inc_internal/maat_ip_plugin.h @@ -57,6 +57,10 @@ int ip_plugin_runtime_get_ex_data(void *ip_plugin_runtime, const struct ip_addr long long ip_plugin_runtime_update_err_count(void *ip_plugin_runtime); +void ip_plugin_runtime_scan_inc(void *ip_plugin_rt, int thread_id); + +long long ip_plugin_runtime_scan_count(void *ip_plugin_runtime); + #ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_ipport_plugin.h b/src/inc_internal/maat_ipport_plugin.h index 83d1fc8..6dc2452 100644 --- a/src/inc_internal/maat_ipport_plugin.h +++ b/src/inc_internal/maat_ipport_plugin.h @@ -59,6 +59,10 @@ int ipport_plugin_runtime_get_ex_data(void *ipport_plugin_runtime, const struct long long ipport_plugin_runtime_update_err_count(void *ipport_plugin_runtime); +void ipport_plugin_runtime_scan_inc(void *ipport_plugin_rt, int thread_id); + +long long ipport_plugin_runtime_scan_count(void *ipport_plugin_runtime); + #ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_plugin.h b/src/inc_internal/maat_plugin.h index ee1be29..bdd4b4a 100644 --- a/src/inc_internal/maat_plugin.h +++ b/src/inc_internal/maat_plugin.h @@ -73,6 +73,10 @@ const char *plugin_runtime_cached_row_get(void *plugin_runtime, size_t index); void *plugin_runtime_get_ex_data(void *plugin_runtime, void *plugin_schema, const char *key, size_t key_len); +void plugin_runtime_scan_inc(void *plugin_runtime, int thread_id); + +long long plugin_runtime_scan_count(void *plugin_runtime); + #ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index 8996ebc..1d4314e 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -179,6 +179,7 @@ struct maat { long long new_version; int is_running; + int thread_counter; pthread_t cfg_mon_thread; pthread_mutex_t background_update_mutex; diff --git a/src/maat_api.c b/src/maat_api.c index d21e648..1e8f77e 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -436,6 +436,25 @@ void maat_free(struct maat *maat_inst) pthread_join(maat_inst->cfg_mon_thread, &ret); } +__thread int _thread_local_tid = -1; +void maat_register_thread(struct maat *maat_inst) +{ + int thread_id = __sync_fetch_and_add(&maat_inst->thread_counter, 1); + assert(thread_id < maat_inst->opts.nr_worker_thread); + + _thread_local_tid = thread_id; +} + +static int _get_tid(struct maat *maat_inst) +{ + if (_thread_local_tid < 0 || + _thread_local_tid >= maat_inst->opts.nr_worker_thread) { + return -1; + } + + return _thread_local_tid; +} + int maat_helper_read_column(const char *table_line, int Nth_column, size_t *column_offset, size_t *column_len) { @@ -870,6 +889,11 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, return NULL; } + int thread_id = _get_tid(maat_inst); + if (thread_id >= 0 && thread_id < maat_inst->opts.nr_worker_thread) { + plugin_runtime_scan_inc(runtime, thread_id); + } + void *ret = NULL; enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id); @@ -901,13 +925,12 @@ int maat_ip_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, return -1; } - int n_ex_data = ip_plugin_runtime_get_ex_data(ip_plugin_rt, ip_addr, - ex_data_array, array_size); - if (n_ex_data < 0) { - return -1; + int thread_id = _get_tid(maat_inst); + if (thread_id >= 0 && thread_id < maat_inst->opts.nr_worker_thread) { + ip_plugin_runtime_scan_inc(ip_plugin_rt, thread_id); } - return n_ex_data; + return ip_plugin_runtime_get_ex_data(ip_plugin_rt, ip_addr, ex_data_array, array_size); } int maat_ipport_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, @@ -929,13 +952,13 @@ int maat_ipport_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, return -1; } - int n_ex_data = ipport_plugin_runtime_get_ex_data(ipport_plugin_rt, ip_addr, port, - ex_data_array, array_size); - if (n_ex_data < 0) { - return -1; + int thread_id = _get_tid(maat_inst); + if (thread_id >= 0 && thread_id < maat_inst->opts.nr_worker_thread) { + ipport_plugin_runtime_scan_inc(ipport_plugin_rt, thread_id); } - return n_ex_data; + return ipport_plugin_runtime_get_ex_data(ipport_plugin_rt, ip_addr, port, + ex_data_array, array_size); } int maat_fqdn_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, @@ -957,13 +980,12 @@ int maat_fqdn_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, return -1; } - int n_ex_data = fqdn_plugin_runtime_get_ex_data(fqdn_plugin_rt, fqdn, - ex_data_array, array_size); - if (n_ex_data < 0) { - return -1; + int thread_id = _get_tid(maat_inst); + if (thread_id >= 0 && thread_id < maat_inst->opts.nr_worker_thread) { + fqdn_plugin_runtime_scan_inc(fqdn_plugin_rt, thread_id); } - return n_ex_data; + return fqdn_plugin_runtime_get_ex_data(fqdn_plugin_rt, fqdn, ex_data_array, array_size); } int maat_bool_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, @@ -985,13 +1007,13 @@ int maat_bool_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, return -1; } - int n_ex_data = bool_plugin_runtime_get_ex_data(bool_plugin_rt, item_ids, n_item, - ex_data_array, array_size); - if (n_ex_data < 0) { - return -1; + int thread_id = _get_tid(maat_inst); + if (thread_id >= 0 && thread_id < maat_inst->opts.nr_worker_thread) { + bool_plugin_runtime_scan_inc(bool_plugin_rt, thread_id); } - return n_ex_data; + return bool_plugin_runtime_get_ex_data(bool_plugin_rt, item_ids, n_item, + ex_data_array, array_size); } static inline int scan_status_should_compile_NOT(struct maat_state *state) diff --git a/src/maat_bool_plugin.c b/src/maat_bool_plugin.c index a3f38ad..ae911eb 100644 --- a/src/maat_bool_plugin.c +++ b/src/maat_bool_plugin.c @@ -39,6 +39,7 @@ struct bool_plugin_runtime { long long rule_num; long long update_err_cnt; + long long *scan_cnt; }; /* bool plugin schema API */ @@ -198,6 +199,7 @@ void *bool_plugin_runtime_new(void *bool_plugin_schema, size_t max_thread_num, bool_plugin_rt->n_worker_thread = max_thread_num; bool_plugin_rt->ref_garbage_bin = garbage_bin; bool_plugin_rt->logger = logger; + bool_plugin_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); return bool_plugin_rt; } @@ -219,6 +221,11 @@ void bool_plugin_runtime_free(void *bool_plugin_runtime) bool_plugin_rt->ex_data_rt = NULL; } + if (bool_plugin_rt->scan_cnt != NULL) { + alignment_int64_array_free(bool_plugin_rt->scan_cnt); + bool_plugin_rt->scan_cnt = NULL; + } + FREE(bool_plugin_rt); } @@ -568,4 +575,28 @@ long long bool_plugin_runtime_update_err_count(void *bool_plugin_runtime) struct bool_plugin_runtime *bool_plugin_rt = (struct bool_plugin_runtime *)bool_plugin_runtime; return bool_plugin_rt->update_err_cnt; +} + +void bool_plugin_runtime_scan_inc(void *bool_plugin_runtime, int thread_id) +{ + if (NULL == bool_plugin_runtime || thread_id < 0) { + return; + } + + struct bool_plugin_runtime *bool_plugin_rt = (struct bool_plugin_runtime *)bool_plugin_runtime; + alignment_int64_array_add(bool_plugin_rt->scan_cnt, thread_id, 1); +} + +long long bool_plugin_runtime_scan_count(void *bool_plugin_runtime) +{ + if (NULL == bool_plugin_runtime) { + return 0; + } + + struct bool_plugin_runtime *bool_plugin_rt = (struct bool_plugin_runtime *)bool_plugin_runtime; + long long sum = alignment_int64_array_sum(bool_plugin_rt->scan_cnt, + bool_plugin_rt->n_worker_thread); + alignment_int64_array_reset(bool_plugin_rt->scan_cnt, bool_plugin_rt->n_worker_thread); + + return sum; }
\ No newline at end of file diff --git a/src/maat_fqdn_plugin.c b/src/maat_fqdn_plugin.c index 71c0fa9..dc47f50 100644 --- a/src/maat_fqdn_plugin.c +++ b/src/maat_fqdn_plugin.c @@ -40,6 +40,7 @@ struct fqdn_plugin_runtime { long long rule_num; long long update_err_cnt; + long long *scan_cnt; }; void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, @@ -197,6 +198,7 @@ void *fqdn_plugin_runtime_new(void *fqdn_plugin_schema, size_t max_thread_num, fqdn_plugin_rt->n_worker_thread = max_thread_num; fqdn_plugin_rt->ref_garbage_bin = garbage_bin; fqdn_plugin_rt->logger = logger; + fqdn_plugin_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); return fqdn_plugin_rt; } @@ -218,6 +220,11 @@ void fqdn_plugin_runtime_free(void *fqdn_plugin_runtime) fqdn_plugin_rt->ex_data_rt = NULL; } + if (fqdn_plugin_rt->scan_cnt != NULL) { + alignment_int64_array_free(fqdn_plugin_rt->scan_cnt); + fqdn_plugin_rt->scan_cnt = NULL; + } + FREE(fqdn_plugin_rt); } @@ -582,4 +589,28 @@ long long fqdn_plugin_runtime_update_err_count(void *fqdn_plugin_runtime) struct fqdn_plugin_runtime *fqdn_plugin_rt = (struct fqdn_plugin_runtime *)fqdn_plugin_runtime; return fqdn_plugin_rt->update_err_cnt; +} + +void fqdn_plugin_runtime_scan_inc(void *fqdn_plugin_runtime, int thread_id) +{ + if (NULL == fqdn_plugin_runtime || thread_id < 0) { + return; + } + + struct fqdn_plugin_runtime *fqdn_plugin_rt = (struct fqdn_plugin_runtime *)fqdn_plugin_runtime; + alignment_int64_array_add(fqdn_plugin_rt->scan_cnt, thread_id, 1); +} + +long long fqdn_plugin_runtime_scan_count(void *fqdn_plugin_runtime) +{ + if (NULL == fqdn_plugin_runtime) { + return 0; + } + + struct fqdn_plugin_runtime *fqdn_plugin_rt = (struct fqdn_plugin_runtime *)fqdn_plugin_runtime; + long long sum = alignment_int64_array_sum(fqdn_plugin_rt->scan_cnt, + fqdn_plugin_rt->n_worker_thread); + alignment_int64_array_reset(fqdn_plugin_rt->scan_cnt, fqdn_plugin_rt->n_worker_thread); + + return sum; }
\ No newline at end of file diff --git a/src/maat_ip_plugin.c b/src/maat_ip_plugin.c index 21bf705..c94703e 100644 --- a/src/maat_ip_plugin.c +++ b/src/maat_ip_plugin.c @@ -43,6 +43,7 @@ struct ip_plugin_runtime { long long rule_num; long long update_err_cnt; + long long *scan_cnt; }; void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, @@ -360,7 +361,8 @@ void *ip_plugin_runtime_new(void *ip_plugin_schema, size_t max_thread_num, ip_plugin_rt->n_worker_thread = max_thread_num; ip_plugin_rt->ref_garbage_bin = garbage_bin; ip_plugin_rt->logger = logger; - + ip_plugin_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); + return ip_plugin_rt; } @@ -381,6 +383,11 @@ void ip_plugin_runtime_free(void *ip_plugin_runtime) ip_plugin_rt->ex_data_rt = NULL; } + if (ip_plugin_rt->scan_cnt != NULL) { + alignment_int64_array_free(ip_plugin_rt->scan_cnt); + ip_plugin_rt->scan_cnt = NULL; + } + FREE(ip_plugin_rt); } @@ -582,4 +589,28 @@ long long ip_plugin_runtime_update_err_count(void *ip_plugin_runtime) struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; return ip_plugin_rt->update_err_cnt; +} + +void ip_plugin_runtime_scan_inc(void *ip_plugin_runtime, int thread_id) +{ + if (NULL == ip_plugin_runtime || thread_id < 0) { + return; + } + + struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; + alignment_int64_array_add(ip_plugin_rt->scan_cnt, thread_id, 1); +} + +long long ip_plugin_runtime_scan_count(void *ip_plugin_runtime) +{ + if (NULL == ip_plugin_runtime) { + return 0; + } + + struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; + long long sum = alignment_int64_array_sum(ip_plugin_rt->scan_cnt, + ip_plugin_rt->n_worker_thread); + alignment_int64_array_reset(ip_plugin_rt->scan_cnt, ip_plugin_rt->n_worker_thread); + + return sum; }
\ No newline at end of file diff --git a/src/maat_ipport_plugin.c b/src/maat_ipport_plugin.c index c4d05e1..4220048 100644 --- a/src/maat_ipport_plugin.c +++ b/src/maat_ipport_plugin.c @@ -66,6 +66,7 @@ struct ipport_plugin_runtime { long long rule_num; long long update_err_cnt; + long long *scan_cnt; }; void *ipport_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, @@ -223,7 +224,8 @@ void *ipport_plugin_runtime_new(void *ipport_plugin_schema, size_t max_thread_nu ipport_plugin_rt->n_worker_thread = max_thread_num; ipport_plugin_rt->ref_garbage_bin = garbage_bin; ipport_plugin_rt->logger = logger; - + ipport_plugin_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); + return ipport_plugin_rt; } @@ -244,6 +246,11 @@ void ipport_plugin_runtime_free(void *ipport_plugin_runtime) ipport_plugin_rt->ex_data_rt = NULL; } + if (ipport_plugin_rt->scan_cnt != NULL) { + alignment_int64_array_free(ipport_plugin_rt->scan_cnt); + ipport_plugin_rt->scan_cnt = NULL; + } + FREE(ipport_plugin_rt); } @@ -621,4 +628,28 @@ long long ipport_plugin_runtime_update_err_count(void *ipport_plugin_runtime) struct ipport_plugin_runtime *ipport_plugin_rt = (struct ipport_plugin_runtime *)ipport_plugin_runtime; return ipport_plugin_rt->update_err_cnt; +} + +void ipport_plugin_runtime_scan_inc(void *ipport_plugin_runtime, int thread_id) +{ + if (NULL == ipport_plugin_runtime || thread_id < 0) { + return; + } + + struct ipport_plugin_runtime *ipport_plugin_rt = (struct ipport_plugin_runtime *)ipport_plugin_runtime; + alignment_int64_array_add(ipport_plugin_rt->scan_cnt, thread_id, 1); +} + +long long ipport_plugin_runtime_scan_count(void *ipport_plugin_runtime) +{ + if (NULL == ipport_plugin_runtime) { + return 0; + } + + struct ipport_plugin_runtime *ipport_plugin_rt = (struct ipport_plugin_runtime *)ipport_plugin_runtime; + long long sum = alignment_int64_array_sum(ipport_plugin_rt->scan_cnt, + ipport_plugin_rt->n_worker_thread); + alignment_int64_array_reset(ipport_plugin_rt->scan_cnt, ipport_plugin_rt->n_worker_thread); + + return sum; }
\ No newline at end of file diff --git a/src/maat_plugin.c b/src/maat_plugin.c index ea4b520..12a1e48 100644 --- a/src/maat_plugin.c +++ b/src/maat_plugin.c @@ -18,6 +18,7 @@ #include "maat_plugin.h" #include "maat_limits.h" #include "maat_table.h" +#include "alignment.h" #define MODULE_PLUGIN module_name_str("maat.plugin") @@ -33,12 +34,14 @@ struct plugin_callback_schema { struct plugin_runtime { struct ex_data_runtime *ex_data_rt; + size_t n_worker_thread; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; long long rule_num; long long acc_line_num; long long update_err_cnt; + long long *scan_cnt; }; enum plugin_key_type { @@ -314,8 +317,10 @@ void *plugin_runtime_new(void *plugin_schema, size_t max_thread_num, &(schema->container_schema)); } + plugin_rt->n_worker_thread = max_thread_num; plugin_rt->ref_garbage_bin = garbage_bin; plugin_rt->logger = logger; + plugin_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); return plugin_rt; } @@ -332,6 +337,11 @@ void plugin_runtime_free(void *plugin_runtime) plugin_rt->ex_data_rt = NULL; } + if (plugin_rt->scan_cnt != NULL) { + alignment_int64_array_free(plugin_rt->scan_cnt); + plugin_rt->scan_cnt = NULL; + } + FREE(plugin_rt); } @@ -673,4 +683,28 @@ void *plugin_runtime_get_ex_data(void *plugin_runtime, void *plugin_schema, } return ex_data_runtime_get_ex_data_by_key(plugin_rt->ex_data_rt, key, key_len); +} + +void plugin_runtime_scan_inc(void *plugin_runtime, int thread_id) +{ + if (NULL == plugin_runtime || thread_id < 0) { + return; + } + + struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime; + alignment_int64_array_add(plugin_rt->scan_cnt, thread_id, 1); +} + +long long plugin_runtime_scan_count(void *plugin_runtime) +{ + if (NULL == plugin_runtime) { + return 0; + } + + struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime; + long long sum = alignment_int64_array_sum(plugin_rt->scan_cnt, + plugin_rt->n_worker_thread); + alignment_int64_array_reset(plugin_rt->scan_cnt, plugin_rt->n_worker_thread); + + return sum; }
\ No newline at end of file diff --git a/src/maat_stat.c b/src/maat_stat.c index 19b3b9f..333639c 100644 --- a/src/maat_stat.c +++ b/src/maat_stat.c @@ -379,6 +379,10 @@ static void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on) fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_RULE_NUM], rule_num); total_rule_num += rule_num; + long long scan_cnt = table_manager_runtime_scan_count(stat->ref_tbl_mgr, i); + fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_SCAN_CNT], scan_cnt); + total_scan_cnt += scan_cnt; + if (table_type == TABLE_TYPE_PLUGIN || table_type == TABLE_TYPE_IP_PLUGIN || table_type == TABLE_TYPE_IPPORT_PLUGIN || table_type == TABLE_TYPE_BOOL_PLUGIN || table_type == TABLE_TYPE_FQDN_PLUGIN) { @@ -406,10 +410,6 @@ static void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on) total_scan_cpu_time += scan_cpu_time; } - long long scan_cnt = table_manager_runtime_scan_count(stat->ref_tbl_mgr, i); - fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_SCAN_CNT], scan_cnt); - total_scan_cnt += scan_cnt; - long long hit_cnt = table_manager_runtime_hit_count(stat->ref_tbl_mgr, i); fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_HIT_CNT], hit_cnt); total_hit_cnt += hit_cnt; diff --git a/src/maat_table.c b/src/maat_table.c index e08ee43..a8bf0df 100644 --- a/src/maat_table.c +++ b/src/maat_table.c @@ -194,6 +194,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = plugin_runtime_free, .update_runtime = plugin_runtime_update, .commit_runtime = plugin_runtime_commit, + .scan_count = plugin_runtime_scan_count, .rule_count = plugin_runtime_rule_count, .update_err_count = plugin_runtime_update_err_count }, @@ -205,6 +206,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = ip_plugin_runtime_free, .update_runtime = ip_plugin_runtime_update, .commit_runtime = ip_plugin_runtime_commit, + .scan_count = ip_plugin_runtime_scan_count, .rule_count = ip_plugin_runtime_rule_count, .update_err_count = ip_plugin_runtime_update_err_count }, @@ -217,6 +219,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .update_runtime = ipport_plugin_runtime_update, .commit_runtime = ipport_plugin_runtime_commit, .rule_count = ipport_plugin_runtime_rule_count, + .scan_count = ipport_plugin_runtime_scan_count, .update_err_count = ipport_plugin_runtime_update_err_count }, { @@ -227,6 +230,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = fqdn_plugin_runtime_free, .update_runtime = fqdn_plugin_runtime_update, .commit_runtime = fqdn_plugin_runtime_commit, + .scan_count = fqdn_plugin_runtime_scan_count, .rule_count = fqdn_plugin_runtime_rule_count, .update_err_count = fqdn_plugin_runtime_update_err_count }, @@ -238,6 +242,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .free_runtime = bool_plugin_runtime_free, .update_runtime = bool_plugin_runtime_update, .commit_runtime = bool_plugin_runtime_commit, + .scan_count = bool_plugin_runtime_scan_count, .rule_count = bool_plugin_runtime_rule_count, .update_err_count = bool_plugin_runtime_update_err_count }, diff --git a/test/maat_framework_perf_gtest.cpp b/test/maat_framework_perf_gtest.cpp index 0dab156..1f636e3 100644 --- a/test/maat_framework_perf_gtest.cpp +++ b/test/maat_framework_perf_gtest.cpp @@ -433,7 +433,8 @@ void *perf_string_scan_thread(void *arg) struct maat_state *state = maat_state_new(maat_inst, param->thread_id); int table_id = maat_get_table_id(maat_inst, table_name); - + maat_register_thread(maat_inst); + clock_gettime(CLOCK_MONOTONIC, &start); for (int i = 0; i < param->test_count; i++) { int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data), @@ -474,6 +475,98 @@ void *perf_string_update_thread(void *arg) return is_all_hit; } +TEST_F(MaatPerfStringScan, LiteralMultiThread) { + const char *table_name = "EXPR_LITERAL_PERF_CONFIG"; + struct maat *maat_inst = MaatPerfStringScan::_shared_maat_inst; + + int table_id = maat_get_table_id(maat_inst, table_name); + ASSERT_GT(table_id, 0); + + pthread_t threads[PERF_THREAD_NUM + 1]; + struct thread_param thread_params[PERF_THREAD_NUM + 1]; + int i = 0; + int *is_all_hit = NULL; + + for (i = 0; i < PERF_THREAD_NUM + 1; i++) { + thread_params[i].maat_inst = maat_inst; + thread_params[i].thread_id = i; + thread_params[i].table_name = table_name; + thread_params[i].test_count = PERF_SCAN_COUNT; + thread_params[i].time_elapse_ms = 0; + thread_params[i].logger = logger; + + if (i < PERF_THREAD_NUM) { + pthread_create(&threads[i], NULL, perf_string_scan_thread, thread_params+i); + } else { + thread_params[i].test_count = 0; + pthread_create(&threads[i], NULL, perf_string_update_thread, thread_params+i); + } + } + + long long time_elapse_ms = 0; + long long scan_count = 0; + long long scan_per_second = 0; + for (i = 0; i < PERF_THREAD_NUM + 1; i++) { + pthread_join(threads[i], (void **)&is_all_hit); + time_elapse_ms += thread_params[i].time_elapse_ms; + scan_count += thread_params[i].test_count; + + EXPECT_EQ(*is_all_hit, 1); + *is_all_hit = 0; + free(is_all_hit); + } + scan_per_second = scan_count * 1000 / time_elapse_ms; + log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST, + "StringScan match rate on %d-threads speed %lld lookups/s/thread", + PERF_THREAD_NUM, scan_per_second); +} + +class MaatPerfRegexScan : public testing::Test +{ +protected: + static void SetUpTestCase() { + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"}," + "{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + logger = log_handle_create("./maat_framework_perf_gtest.log", 0); + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger); + if (ret < 0) { + log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_stat_file(opts, "./stat.log"); + maat_options_set_perf_on(opts); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); + maat_options_set_accept_tags(opts, accept_tags); + maat_options_set_caller_thread_number(opts, 5); + + _shared_maat_inst = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_inst) { + log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, + "[%s:%d] create maat instance in MaatRegexScan failed.", + __FUNCTION__, __LINE__); + } + } + + static void TearDownTestCase() { + maat_free(_shared_maat_inst); + log_handle_destroy(logger); + } + + static struct log_handle *logger; + static struct maat *_shared_maat_inst; +}; + +struct maat *MaatPerfRegexScan::_shared_maat_inst; +struct log_handle *MaatPerfRegexScan::logger; + void *perf_regex_scan_thread(void *arg) { struct thread_param *param = (struct thread_param *)arg; @@ -487,7 +580,8 @@ void *perf_regex_scan_thread(void *arg) struct maat_state *state = maat_state_new(maat_inst, param->thread_id); int table_id = maat_get_table_id(maat_inst, table_name); - + maat_register_thread(maat_inst); + clock_gettime(CLOCK_MONOTONIC, &start); for (int i = 0; i < param->test_count; i++) { int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data), @@ -528,75 +622,54 @@ void *perf_regex_update_thread(void *arg) return is_all_hit; } -void *perf_ip_scan_thread(void *arg) -{ - struct thread_param *param = (struct thread_param *)arg; - struct maat *maat_inst = param->maat_inst; - const char *table_name = param->table_name; - struct timespec start, end; - char ip_str[32] = "10.0.0.1"; - uint32_t ip_addr; - uint16_t port = htons(65530); +TEST_F(MaatPerfRegexScan, RegexMultiThread) { + const char *table_name = "EXPR_REGEX_PERF_CONFIG"; + struct maat *maat_inst = MaatPerfRegexScan::_shared_maat_inst; - int ret = inet_pton(AF_INET, ip_str, &ip_addr); - EXPECT_EQ(ret, 1); - - int hit_times = 0; - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - struct maat_state *state = maat_state_new(maat_inst, param->thread_id); int table_id = maat_get_table_id(maat_inst, table_name); + ASSERT_GT(table_id, 0); - clock_gettime(CLOCK_MONOTONIC, &start); - for (int i = 0; i < param->test_count; i++) { - int ret = maat_scan_ipv4(maat_inst, table_id, ip_addr, port, 6, - results, ARRAY_SIZE, &n_hit_result, state); - if (ret == MAAT_SCAN_HIT) { - hit_times++; + pthread_t threads[PERF_THREAD_NUM + 1]; + struct thread_param thread_params[PERF_THREAD_NUM + 1]; + int i = 0; + int *is_all_hit = NULL; + + for (i = 0; i < PERF_THREAD_NUM + 1; i++) { + thread_params[i].maat_inst = maat_inst; + thread_params[i].thread_id = i; + thread_params[i].table_name = table_name; + thread_params[i].test_count = PERF_SCAN_COUNT; + thread_params[i].time_elapse_ms = 0; + thread_params[i].logger = logger; + + if (i < PERF_THREAD_NUM) { + pthread_create(&threads[i], NULL, perf_regex_scan_thread, thread_params+i); + } else { + thread_params[i].test_count = 0; + pthread_create(&threads[i], NULL, perf_regex_update_thread, thread_params+i); } - maat_state_reset(state); } - clock_gettime(CLOCK_MONOTONIC, &end); - - param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000; - int *is_all_hit = ALLOC(int, 1); - *is_all_hit = (hit_times == param->test_count ? 1 : 0); - log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST, - "thread_id:%d ip_scan time_elapse:%lldms hit_times:%d", - param->thread_id, param->time_elapse_ms, hit_times); - return is_all_hit; -} -void *perf_ip_update_thread(void *arg) -{ - struct thread_param *param = (struct thread_param *)arg; - struct maat *maat_inst = param->maat_inst; - const char *table_name = param->table_name; - const int CMD_EXPR_NUM = 10; - char ip_str[10][32] = { - "10.0.7.101", - "10.0.7.102", - "10.0.7.103", - "10.0.7.104", - "10.0.7.105", - "10.0.7.106", - "10.0.7.107", - "10.0.7.108", - "10.0.7.109", - "10.0.7.110"}; + long long time_elapse_ms = 0; + long long scan_count = 0; + long long scan_per_second = 0; + for (i = 0; i < PERF_THREAD_NUM + 1; i++) { + pthread_join(threads[i], (void **)&is_all_hit); + time_elapse_ms += thread_params[i].time_elapse_ms; + scan_count += thread_params[i].test_count; - uint16_t port = 65530; - for (int i = 0; i < CMD_EXPR_NUM; i++) { - test_add_ip_command(maat_inst, table_name, ip_str[i], port); - sleep(1); + EXPECT_EQ(*is_all_hit, 1); + *is_all_hit = 0; + free(is_all_hit); } - - int *is_all_hit = ALLOC(int, 1); - *is_all_hit = 1; + scan_per_second = scan_count * 1000 / time_elapse_ms; - return is_all_hit; + log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST, + "RegexScan match rate on %d-threads speed %lld lookups/s/thread", + PERF_THREAD_NUM, scan_per_second); } + void *perf_integer_scan_thread(void *arg) { struct thread_param *param = (struct thread_param *)arg; @@ -609,6 +682,7 @@ void *perf_integer_scan_thread(void *arg) struct maat_state *state = maat_state_new(maat_inst, param->thread_id); int table_id = maat_get_table_id(maat_inst, table_name); + maat_register_thread(maat_inst); clock_gettime(CLOCK_MONOTONIC, &start); for (int i = 0; i < param->test_count; i++) { @@ -649,152 +723,6 @@ void *perf_integer_update_thread(void *arg) return is_all_hit; } -void *perf_flag_scan_thread(void *arg) -{ - struct thread_param *param = (struct thread_param *)arg; - struct maat *maat_inst = param->maat_inst; - const char *table_name = param->table_name; - struct timespec start, end; - int hit_times = 0; - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - long long scan_data = 15; - - struct maat_state *state = maat_state_new(maat_inst, param->thread_id); - int table_id = maat_get_table_id(maat_inst, table_name); - - clock_gettime(CLOCK_MONOTONIC, &start); - for (int i = 0; i < param->test_count; i++) { - int ret = maat_scan_flag(maat_inst, table_id, scan_data, results, - ARRAY_SIZE, &n_hit_result, state); - if (ret == MAAT_SCAN_HIT) { - hit_times++; - } - maat_state_reset(state); - } - clock_gettime(CLOCK_MONOTONIC, &end); - - param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + - (end.tv_nsec - start.tv_nsec) / 1000000; - int *is_all_hit = ALLOC(int, 1); - *is_all_hit = (hit_times == param->test_count ? 1 : 0); - log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST, - "thread_id:%d flag_scan time_elapse:%lldms hit_times:%d", - param->thread_id, param->time_elapse_ms, hit_times); - return is_all_hit; -} - -void *perf_flag_update_thread(void *arg) -{ - struct thread_param *param = (struct thread_param *)arg; - struct maat *maat_inst = param->maat_inst; - const char *table_name = param->table_name; - const int CMD_EXPR_NUM = 10; - - for (int i = 0; i < CMD_EXPR_NUM; i++) { - test_add_flag_command(maat_inst, table_name, i, 15); - sleep(1); - } - - int *is_all_hit = ALLOC(int, 1); - *is_all_hit = 1; - - return is_all_hit; -} - -TEST_F(MaatPerfStringScan, LiteralMultiThread) { - const char *table_name = "EXPR_LITERAL_PERF_CONFIG"; - struct maat *maat_inst = MaatPerfStringScan::_shared_maat_inst; - - int table_id = maat_get_table_id(maat_inst, table_name); - ASSERT_GT(table_id, 0); - - pthread_t threads[PERF_THREAD_NUM + 1]; - struct thread_param thread_params[PERF_THREAD_NUM + 1]; - int i = 0; - int *is_all_hit = NULL; - - for (i = 0; i < PERF_THREAD_NUM + 1; i++) { - thread_params[i].maat_inst = maat_inst; - thread_params[i].thread_id = i; - thread_params[i].table_name = table_name; - thread_params[i].test_count = PERF_SCAN_COUNT; - thread_params[i].time_elapse_ms = 0; - thread_params[i].logger = logger; - - if (i < PERF_THREAD_NUM) { - pthread_create(&threads[i], NULL, perf_string_scan_thread, thread_params+i); - } else { - thread_params[i].test_count = 0; - pthread_create(&threads[i], NULL, perf_string_update_thread, thread_params+i); - } - } - - long long time_elapse_ms = 0; - long long scan_count = 0; - long long scan_per_second = 0; - for (i = 0; i < PERF_THREAD_NUM + 1; i++) { - pthread_join(threads[i], (void **)&is_all_hit); - time_elapse_ms += thread_params[i].time_elapse_ms; - scan_count += thread_params[i].test_count; - - EXPECT_EQ(*is_all_hit, 1); - *is_all_hit = 0; - free(is_all_hit); - } - scan_per_second = scan_count * 1000 / time_elapse_ms; - log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST, - "StringScan match rate on %d-threads speed %lld lookups/s/thread", - PERF_THREAD_NUM, scan_per_second); -} - -TEST_F(MaatPerfStringScan, RegexMultiThread) { - const char *table_name = "EXPR_REGEX_PERF_CONFIG"; - struct maat *maat_inst = MaatPerfStringScan::_shared_maat_inst; - - int table_id = maat_get_table_id(maat_inst, table_name); - ASSERT_GT(table_id, 0); - - pthread_t threads[PERF_THREAD_NUM + 1]; - struct thread_param thread_params[PERF_THREAD_NUM + 1]; - int i = 0; - int *is_all_hit = NULL; - - for (i = 0; i < PERF_THREAD_NUM + 1; i++) { - thread_params[i].maat_inst = maat_inst; - thread_params[i].thread_id = i; - thread_params[i].table_name = table_name; - thread_params[i].test_count = PERF_SCAN_COUNT; - thread_params[i].time_elapse_ms = 0; - thread_params[i].logger = logger; - - if (i < PERF_THREAD_NUM) { - pthread_create(&threads[i], NULL, perf_regex_scan_thread, thread_params+i); - } else { - thread_params[i].test_count = 0; - pthread_create(&threads[i], NULL, perf_regex_update_thread, thread_params+i); - } - } - - long long time_elapse_ms = 0; - long long scan_count = 0; - long long scan_per_second = 0; - for (i = 0; i < PERF_THREAD_NUM + 1; i++) { - pthread_join(threads[i], (void **)&is_all_hit); - time_elapse_ms += thread_params[i].time_elapse_ms; - scan_count += thread_params[i].test_count; - - EXPECT_EQ(*is_all_hit, 1); - *is_all_hit = 0; - free(is_all_hit); - } - scan_per_second = scan_count * 1000 / time_elapse_ms; - - log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST, - "RegexScan match rate on %d-threads speed %lld lookups/s/thread", - PERF_THREAD_NUM, scan_per_second); -} - class MaatPerfStreamScan : public testing::Test { protected: @@ -855,6 +783,7 @@ void *perf_stream_scan_thread(void *arg) int table_id = maat_get_table_id(maat_inst, table_name); struct maat_state *state = maat_state_new(maat_inst, param->thread_id); struct maat_stream *sp = maat_stream_new(maat_inst, table_id, state); + maat_register_thread(maat_inst); clock_gettime(CLOCK_MONOTONIC, &start); for (int i = 0; i < param->test_count; i++) { @@ -968,6 +897,76 @@ protected: struct maat *MaatPerfIPScan::_shared_maat_inst; struct log_handle *MaatPerfIPScan::logger; +void *perf_ip_scan_thread(void *arg) +{ + struct thread_param *param = (struct thread_param *)arg; + struct maat *maat_inst = param->maat_inst; + const char *table_name = param->table_name; + struct timespec start, end; + char ip_str[32] = "10.0.0.1"; + uint32_t ip_addr; + uint16_t port = htons(65530); + + int ret = inet_pton(AF_INET, ip_str, &ip_addr); + EXPECT_EQ(ret, 1); + + int hit_times = 0; + long long results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = maat_state_new(maat_inst, param->thread_id); + int table_id = maat_get_table_id(maat_inst, table_name); + maat_register_thread(maat_inst); + + clock_gettime(CLOCK_MONOTONIC, &start); + for (int i = 0; i < param->test_count; i++) { + int ret = maat_scan_ipv4(maat_inst, table_id, ip_addr, port, 6, + results, ARRAY_SIZE, &n_hit_result, state); + if (ret == MAAT_SCAN_HIT) { + hit_times++; + } + maat_state_reset(state); + } + clock_gettime(CLOCK_MONOTONIC, &end); + + param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000; + int *is_all_hit = ALLOC(int, 1); + *is_all_hit = (hit_times == param->test_count ? 1 : 0); + log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST, + "thread_id:%d ip_scan time_elapse:%lldms hit_times:%d", + param->thread_id, param->time_elapse_ms, hit_times); + return is_all_hit; +} + +void *perf_ip_update_thread(void *arg) +{ + struct thread_param *param = (struct thread_param *)arg; + struct maat *maat_inst = param->maat_inst; + const char *table_name = param->table_name; + const int CMD_EXPR_NUM = 10; + char ip_str[10][32] = { + "10.0.7.101", + "10.0.7.102", + "10.0.7.103", + "10.0.7.104", + "10.0.7.105", + "10.0.7.106", + "10.0.7.107", + "10.0.7.108", + "10.0.7.109", + "10.0.7.110"}; + + uint16_t port = 65530; + for (int i = 0; i < CMD_EXPR_NUM; i++) { + test_add_ip_command(maat_inst, table_name, ip_str[i], port); + sleep(1); + } + + int *is_all_hit = ALLOC(int, 1); + *is_all_hit = 1; + + return is_all_hit; +} + TEST_F(MaatPerfIPScan, MultiThread) { const char *table_name = "IP_PERF_CONFIG"; @@ -1155,6 +1154,59 @@ protected: struct maat *MaatPerfFlagScan::_shared_maat_inst; struct log_handle *MaatPerfFlagScan::logger; +void *perf_flag_scan_thread(void *arg) +{ + struct thread_param *param = (struct thread_param *)arg; + struct maat *maat_inst = param->maat_inst; + const char *table_name = param->table_name; + struct timespec start, end; + int hit_times = 0; + long long results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + long long scan_data = 15; + + struct maat_state *state = maat_state_new(maat_inst, param->thread_id); + int table_id = maat_get_table_id(maat_inst, table_name); + + clock_gettime(CLOCK_MONOTONIC, &start); + for (int i = 0; i < param->test_count; i++) { + int ret = maat_scan_flag(maat_inst, table_id, scan_data, results, + ARRAY_SIZE, &n_hit_result, state); + if (ret == MAAT_SCAN_HIT) { + hit_times++; + } + maat_state_reset(state); + } + clock_gettime(CLOCK_MONOTONIC, &end); + + param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + + (end.tv_nsec - start.tv_nsec) / 1000000; + int *is_all_hit = ALLOC(int, 1); + *is_all_hit = (hit_times == param->test_count ? 1 : 0); + log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST, + "thread_id:%d flag_scan time_elapse:%lldms hit_times:%d", + param->thread_id, param->time_elapse_ms, hit_times); + return is_all_hit; +} + +void *perf_flag_update_thread(void *arg) +{ + struct thread_param *param = (struct thread_param *)arg; + struct maat *maat_inst = param->maat_inst; + const char *table_name = param->table_name; + const int CMD_EXPR_NUM = 10; + + for (int i = 0; i < CMD_EXPR_NUM; i++) { + test_add_flag_command(maat_inst, table_name, i, 15); + sleep(1); + } + + int *is_all_hit = ALLOC(int, 1); + *is_all_hit = 1; + + return is_all_hit; +} + TEST_F(MaatPerfFlagScan, MultiThread) { const char *table_name = "FLAG_PERF_CONFIG"; struct maat *maat_inst = MaatPerfFlagScan::_shared_maat_inst; @@ -1296,6 +1348,7 @@ void* perf_fqdn_plugin_scan_thread(void *arg) int i=0, ret=0, hit_times=0; int table_id = maat_get_table_id(maat_inst, param->table_name); + maat_register_thread(maat_inst); memset(&result, 0, sizeof(result)); struct timespec start, end; @@ -1522,6 +1575,7 @@ void* perf_bool_plugin_scan_thread(void *arg) int i=0, ret=0, hit_times=0; int table_id = maat_get_table_id(maat_inst, param->table_name); + maat_register_thread(maat_inst); memset(&result, 0, sizeof(result)); struct timespec start, end; @@ -1711,13 +1765,13 @@ static void *ip_plugin_get_thread(void *arg) struct maat *maat_inst = (struct maat *)arg; int table_id = maat_get_table_id(maat_inst, table_name); + maat_register_thread(maat_inst); struct timespec start, end; struct ip_addr ipv4; ipv4.ip_type = IPv4; - inet_pton(AF_INET, "191.70.72.1", &ipv4.ipv4); - + clock_gettime(CLOCK_MONOTONIC, &start); struct perf_ip_plugin_ud *results[ARRAY_SIZE]; for (i = 0; i < test_times; i++) { @@ -1769,6 +1823,153 @@ TEST_F(MaatPerfFileScan, IPPlugin) { } } +class MaatPerfIPPortPluginScan : public testing::Test +{ +protected: + static void SetUpTestCase() { + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"}," + "{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + logger = log_handle_create("./maat_framework_perf_gtest.log", 0); + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger); + if (ret < 0) { + log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_stat_file(opts, "./stat.log"); + maat_options_set_perf_on(opts); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); + maat_options_set_accept_tags(opts, accept_tags); + maat_options_set_caller_thread_number(opts, 5); + + _shared_maat_inst = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_inst) { + log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, + "[%s:%d] create maat instance in MaatFlagScan failed.", + __FUNCTION__, __LINE__); + } + } + + static void TearDownTestCase() { + maat_free(_shared_maat_inst); + log_handle_destroy(logger); + } + + static struct log_handle *logger; + static struct maat *_shared_maat_inst; +}; + +struct maat *MaatPerfIPPortPluginScan::_shared_maat_inst; +struct log_handle *MaatPerfIPPortPluginScan::logger; + +static void *perf_ipport_plugin_scan_thread(void *arg) +{ + struct thread_param *param = (struct thread_param *)arg; + struct maat *maat_inst = param->maat_inst; + struct perf_ip_plugin_ud *results[ARRAY_SIZE]; + int i = 0, ret = 0, hit_times = 0; + + int table_id = maat_get_table_id(maat_inst, param->table_name); + maat_register_thread(maat_inst); + + struct ip_addr ipv4; + ipv4.ip_type = IPv4; + inet_pton(AF_INET, "192.168.100.1", &ipv4.ipv4); + uint16_t port = htons(215); + + for (i = 0; i < param->test_count; i++) { + ret = maat_ipport_plugin_table_get_ex_data(maat_inst, table_id, &ipv4, + port, (void**)results, 4); + if (ret > 0) { + hit_times++; + } + } + + int *is_all_hit = (int *)malloc(sizeof(int)); + *is_all_hit = (hit_times == param->test_count) ? 1 : 0; + log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST, + "ipport_plugin_get_ex_data hit_times:%d", hit_times); + + return is_all_hit; +} + +void *perf_ipport_plugin_update_thread(void *arg) +{ + struct thread_param *param = (struct thread_param *)arg; + struct maat *maat_inst = param->maat_inst; + const int CMD_EXPR_NUM = 20; + int i = 0; + struct maat_cmd_line line_rule; + char line_buff[1024]; + + for (i = 0; i < CMD_EXPR_NUM; i++) { + line_rule.rule_id = (int)maat_cmd_incrby(maat_inst, "TEST_PLUG_SEQ", 1); + line_rule.table_name = param->table_name; + snprintf(line_buff, 1024, "%lld\t4\t192.168.100.1\t%d\t%d\t1", + line_rule.rule_id, 200+i, 300+i); + line_rule.table_line = line_buff; + line_rule.expire_after = 0; + maat_cmd_set_line(maat_inst, &line_rule); + sleep(WAIT_FOR_EFFECTIVE_S); + } + + int *is_all_hit = (int *)malloc(sizeof(int)); + *is_all_hit = 1; + + return is_all_hit; +} + +TEST_F(MaatPerfIPPortPluginScan, MultiThread) { + struct maat *maat_inst = MaatPerfIPPortPluginScan::_shared_maat_inst; + const char* table_name = "TEST_IPPORT_PLUGIN_WITH_EXDATA"; + int ip_plugin_ex_data_counter = 0; + + int table_id = maat_get_table_id(maat_inst, table_name); + ASSERT_GT(table_id, 0); + + int ret = maat_plugin_table_ex_schema_register(maat_inst, table_name, + perf_ip_plugin_EX_new_cb, + perf_ip_plugin_EX_free_cb, + perf_ip_plugin_EX_dup_cb, + 0, &ip_plugin_ex_data_counter); + ASSERT_TRUE(ret>=0); + + pthread_t threads[PERF_THREAD_NUM + 1]; + struct thread_param thread_params[PERF_THREAD_NUM + 1]; + int i = 0; + int *is_all_hit = NULL; + + for (i = 0; i < PERF_THREAD_NUM + 1; i++) { + thread_params[i].maat_inst = maat_inst; + thread_params[i].thread_id = i; + thread_params[i].table_name = table_name; + thread_params[i].test_count = PERF_SCAN_COUNT; + thread_params[i].time_elapse_ms = 0; + thread_params[i].logger = logger; + + if (i < PERF_THREAD_NUM) { + pthread_create(&threads[i], NULL, perf_ipport_plugin_scan_thread, thread_params+i); + } else { + thread_params[i].test_count = 0; + pthread_create(&threads[i], NULL, perf_ipport_plugin_update_thread, thread_params+i); + } + } + + for (i = 0; i < PERF_THREAD_NUM + 1; i++) { + pthread_join(threads[i], (void **)&is_all_hit); + EXPECT_EQ(*is_all_hit, 1); + *is_all_hit = 0; + free(is_all_hit); + } +} + int main(int argc, char ** argv) { int ret=0; diff --git a/test/maat_json.json b/test/maat_json.json index 0e7601a..81fadff 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -2973,6 +2973,134 @@ "not_flag": 0 } ] + }, + { + "compile_id": 211, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "ip_perf_test", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PERF_CONFIG", + "table_content": { + "addr_type": "ipv4", + "addr_format": "range", + "ip1": "10.0.0.1", + "ip2": "10.0.0.6", + "port_format": "range", + "port1": "65530", + "port2": "65535", + "protocol": 6 + } + } + ], + "not_flag": 0 + } + ] + }, + { + "compile_id": 212, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "integer_perf_test", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "INTEGER_PERF_CONFIG", + "table_type": "interval", + "table_content": { + "low_boundary": 3000, + "up_boundary": 3000 + } + } + ] + } + ] + }, + { + "compile_id": 213, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "expr_perf_test", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_name": "EXPR_LITERAL_PERF_CONFIG", + "table_type": "expr", + "table_content": { + "keywords": "today&yesterday", + "expr_type": "and", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 214, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "user_region": "flag_perf_test", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "flag", + "table_name": "FLAG_PERF_CONFIG", + "table_content": { + "flag": 15, + "flag_mask": 15 + } + } + ] + } + ] + }, + { + "compile_id": 215, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "expr_perf_test", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_name": "EXPR_REGEX_PERF_CONFIG", + "table_type": "expr", + "table_content": { + "keywords": "action=search\\&query=(.*)", + "expr_type": "regex", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] } ], "plugin_table": [ |
