diff options
| author | liuwentan <[email protected]> | 2023-09-12 16:19:22 +0800 |
|---|---|---|
| committer | liuwentan <[email protected]> | 2023-10-09 12:06:29 +0800 |
| commit | 929f0d0cdd50809b0e8889a568ff43e5f6ba94d2 (patch) | |
| tree | e6003ee2c9592f2fee158c4bd1ae444b0da21228 | |
| parent | 68825a78f7256d9a889affe5e21123d91edfe83c (diff) | |
[PATCH]Each virtual table supports up to 8 NOT clauses
| -rw-r--r-- | src/maat_compile.c | 91 | ||||
| -rw-r--r-- | src/maat_redis_monitor.c | 68 | ||||
| -rw-r--r-- | src/maat_virtual.c | 1 | ||||
| -rw-r--r-- | test/maat_json.json | 222 | ||||
| -rw-r--r-- | test/table_info.conf | 6 |
5 files changed, 341 insertions, 47 deletions
diff --git a/src/maat_compile.c b/src/maat_compile.c index 9e28c17..1b858d0 100644 --- a/src/maat_compile.c +++ b/src/maat_compile.c @@ -83,6 +83,12 @@ struct literal_clause { UT_hash_handle hh; }; +struct vtable_not_clause { + int vtable_id; + int not_clause_num; + UT_hash_handle hh; +}; + /* compile_runtime and group2compile_runtime share compile_hash_map */ struct compile_runtime { struct bool_matcher *bm; @@ -100,10 +106,11 @@ struct compile_runtime { }; struct group2compile_runtime { - long long not_clause_num; + long long not_clause_cnt; long long rule_num; long long update_err_cnt; struct compile_runtime *ref_compile_rt; + struct vtable_not_clause *not_clause_hash; }; struct maat_clause { @@ -683,6 +690,7 @@ void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num, } struct group2compile_runtime *g2c_rt = ALLOC(struct group2compile_runtime, 1); + g2c_rt->not_clause_hash = NULL; return g2c_rt; } @@ -700,6 +708,16 @@ void group2compile_runtime_free(void *g2c_runtime) return; } + struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime; + if (g2c_rt->not_clause_hash != NULL) { + struct vtable_not_clause *not_clause = NULL, *tmp_not_clause = NULL; + HASH_ITER(hh, g2c_rt->not_clause_hash, not_clause, tmp_not_clause) { + HASH_DEL(g2c_rt->not_clause_hash, not_clause); + FREE(not_clause); + } + } + assert(g2c_rt->not_clause_hash == NULL); + FREE(g2c_runtime); } @@ -1901,7 +1919,52 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, return 0; } -#define MAX_NOT_CLAUSE_NUM 16 + +#define MAX_NOT_CLAUSE_NUM 8 +int validate_vtable_not_clause(struct group2compile_runtime *g2c_rt, + struct table_manager *tbl_mgr, int vtable_id, + int is_valid, struct log_handle *logger) +{ + enum table_type table_type = table_manager_get_table_type(tbl_mgr, vtable_id); + if (table_type != TABLE_TYPE_VIRTUAL) { + log_fatal(logger, MODULE_COMPILE, + "[%s:%d]table(table_id:%d) is not virtual table, can't own NOT clause.", + __FUNCTION__, __LINE__, vtable_id); + return -1; + } + + struct vtable_not_clause *not_clause = NULL; + HASH_FIND_INT(g2c_rt->not_clause_hash, &vtable_id, not_clause); + + if (0 == is_valid) { + //delete + if (NULL == not_clause || 0 == not_clause->not_clause_num) { + return 0; + } else { + not_clause->not_clause_num--; + } + } else { + //add + if (NULL == not_clause) { + not_clause = ALLOC(struct vtable_not_clause, 1); + not_clause->vtable_id = vtable_id; + not_clause->not_clause_num++; + HASH_ADD_INT(g2c_rt->not_clause_hash, vtable_id, not_clause); + } else { + if (not_clause->not_clause_num >= MAX_NOT_CLAUSE_NUM) { + const char *table_name = table_manager_get_table_name(tbl_mgr, vtable_id); + log_fatal(logger, MODULE_COMPILE, + "[%s:%d]virtual table:<%s> NOT clause num exceed maximum:%d", + __FUNCTION__, __LINE__, table_name, MAX_NOT_CLAUSE_NUM); + return -1; + } + not_clause->not_clause_num++; + } + } + + return 0; +} + int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *table_name, const char *line, int valid_column) @@ -1932,13 +1995,24 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, return -1; } + if (1 == g2c_item->not_flag) { + ret = validate_vtable_not_clause(g2c_rt, schema->ref_tbl_mgr, g2c_item->vtable_id, + is_valid, compile_rt->logger); + if (ret < 0) { + log_fatal(compile_rt->logger, MODULE_COMPILE, + "[%s:%d]validate NOT clause failed, abandon config:%s", + __FUNCTION__, __LINE__, line); + goto next; + } + } + if (0 == is_valid) { //delete ret = maat_remove_group_from_compile(compile_rt->cfg_hash, g2c_item, compile_rt->logger); if (0 == ret) { if (g2c_item->not_flag) { - g2c_rt->not_clause_num--; + g2c_rt->not_clause_cnt--; } g2c_rt->rule_num--; } else { @@ -1950,20 +2024,15 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, compile_rt->logger); if (0 == ret) { if (g2c_item->not_flag) { - g2c_rt->not_clause_num++; + g2c_rt->not_clause_cnt++; } g2c_rt->rule_num++; } else { g2c_rt->update_err_cnt++; } } - - if (g2c_rt->not_clause_num > MAX_NOT_CLAUSE_NUM) { - log_fatal(compile_rt->logger, MODULE_COMPILE, - "[%s:%d] NOT clause num:%lld exceed maximum:%d which may affect scanning performance", - __FUNCTION__, __LINE__, g2c_rt->not_clause_num, MAX_NOT_CLAUSE_NUM); - } +next: group2compile_item_free(g2c_item); return ret; } @@ -1975,7 +2044,7 @@ long long group2compile_runtime_not_clause_count(void *g2c_runtime) } struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime; - return g2c_rt->not_clause_num; + return g2c_rt->not_clause_cnt; } long long group2compile_runtime_rule_count(void *g2c_runtime) diff --git a/src/maat_redis_monitor.c b/src/maat_redis_monitor.c index fb40936..d446707 100644 --- a/src/maat_redis_monitor.c +++ b/src/maat_redis_monitor.c @@ -43,11 +43,11 @@ const char *foreign_key_prefix = "__FILE_"; const char *mr_op_str[] = {"DEL", "ADD", "RENEW_TIMEOUT"}; -#define POSSIBLE_REDIS_REPLY_SIZE 2 +#define REDIS_REPLY_SIZE 2 struct expected_reply { int s_rule_seq; int possible_reply_num; - redisReply possible_replies[POSSIBLE_REDIS_REPLY_SIZE]; + redisReply possible_replies[REDIS_REPLY_SIZE]; }; static char *get_foreign_cont_filename(const char *table_name, long long rule_id, @@ -867,7 +867,7 @@ static void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, log_error(logger, MODULE_REDIS_MONITOR, "[%s:%d] Get %s,%lld foreign key %s content failed, redis server error", __FUNCTION__, __LINE__, - rule_list[track[i].rule_idx].table_name, + rule_list[track[i].rule_idx].table_name, rule_list[track[i].rule_idx].rule_id, rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key); break; @@ -886,15 +886,15 @@ static void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, FILE *fp = fopen(s_rule->f_keys[track[i].foreign_idx].filename, "w"); if (NULL == fp) { log_error(logger, MODULE_REDIS_MONITOR, - "[%s:%d] Write foreign content failed: fopen %s error", - __FUNCTION__, __LINE__, s_rule->f_keys[track[i].foreign_idx].filename); + "[%s:%d] Write foreign content failed: fopen %s error", __FUNCTION__, + __LINE__, s_rule->f_keys[track[i].foreign_idx].filename); } else { fwrite(reply->str, 1, reply->len, fp); fclose(fp); fp = NULL; if (1 == print_fn) { - printf("[%s:%d] Written foreign content %s\n", - __FUNCTION__, __LINE__, s_rule->f_keys[track[i].foreign_idx].filename); + printf("[%s:%d] Written foreign content %s\n", __FUNCTION__, + __LINE__, s_rule->f_keys[track[i].foreign_idx].filename); } } } @@ -920,7 +920,7 @@ void maat_get_foreign_conts(redisContext *c, struct serial_rule *rule_list, } } -static int invalidate_line(char *line, int column_seq) +static int validate_line(char *line, int column_seq) { if (NULL == line || column_seq < 0) { return -1; @@ -957,7 +957,8 @@ void maat_rewrite_table_line_with_foreign(struct serial_rule *s_rule) strncat(pos_rewrite_line, pos_origin_line, origin_column - pos_origin_line); pos_rewrite_line += origin_column - pos_origin_line; pos_origin_line = origin_column+origin_column_size; - strncat(pos_rewrite_line, s_rule->f_keys[i].filename, strlen(s_rule->f_keys[i].filename)); + strncat(pos_rewrite_line, s_rule->f_keys[i].filename, + strlen(s_rule->f_keys[i].filename)); pos_rewrite_line += strlen(s_rule->f_keys[i].filename); } @@ -971,7 +972,7 @@ static void expected_reply_add(struct expected_reply* expected, int s_rule_seq, int type, long long integer) { int i = expected->possible_reply_num; - assert(i < POSSIBLE_REDIS_REPLY_SIZE); + assert(i < REDIS_REPLY_SIZE); expected->s_rule_seq = s_rule_seq; expected->possible_replies[i].type = type; expected->possible_replies[i].integer = integer; @@ -1049,7 +1050,7 @@ const char* lua_exec_done= "return maat_version;"; static redisReply* exec_serial_rule_end(redisContext *c, const char *transaction_list, long long server_time, int renew_allowed, - struct expected_reply *expect_reply, size_t *cnt) + struct expected_reply *expect_reply, size_t *cnt) { redisReply *data_reply = NULL; @@ -1061,11 +1062,8 @@ static redisReply* exec_serial_rule_end(redisContext *c, const char *transaction if (strlen(transaction_list) > 0) { data_reply = maat_wrap_redis_command(c, "eval %s 4 MAAT_VERSION %s %s %s %lld", - lua_exec_done, - mr_status_sset, - mr_version_sset, - transaction_list, - server_time); + lua_exec_done, mr_status_sset, mr_version_sset, + transaction_list, server_time); freeReplyObject(data_reply); data_reply = NULL; expected_reply_add(expect_reply + *cnt, -1, REDIS_REPLY_INTEGER, 0); @@ -1077,10 +1075,10 @@ static redisReply* exec_serial_rule_end(redisContext *c, const char *transaction return data_reply; } -static void exec_serial_rule(redisContext *c, const char *transaction_list, - struct serial_rule *s_rule, size_t rule_num, - struct expected_reply *expect_reply, size_t *cnt, - size_t offset, int renew_allowed) +static void exec_serial_rule(redisContext *c, const char *transaction_list, + struct serial_rule *s_rule, size_t rule_num, + struct expected_reply *expect_reply, size_t *cnt, + size_t offset, int renew_allowed) { size_t i = 0; size_t append_cmd_cnt = 0; @@ -1235,7 +1233,7 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, char transaction_list[NAME_MAX * 2] = {0}; long long transaction_version = 0; long long transaction_finished_version = 0; - size_t max_multi_cmd_num = MAX_REDIS_OP_PER_SRULE * serial_rule_num + 2;// 2 for operation in exec_serial_rule_end() + size_t max_multi_cmd_num = MAX_REDIS_OP_PER_SRULE * serial_rule_num + 2; // 2 for operation in exec_serial_rule_end() struct expected_reply *expected_reply = ALLOC(struct expected_reply, max_multi_cmd_num); for (i = 0; i < serial_rule_num; i++) { @@ -1456,7 +1454,8 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, //authorized to write if (mr_ctx->write_ctx != NULL && mr_ctx->write_ctx->err == 0) { //For thread safe, deliberately use redis_read_ctx but not redis_write_ctx. - if (1 == redlock_try_lock(mr_ctx->read_ctx, mr_expire_lock, mr_expire_lock_timeout_ms)) { + if (1 == redlock_try_lock(mr_ctx->read_ctx, mr_expire_lock, + mr_expire_lock_timeout_ms)) { check_maat_expiration(mr_ctx->read_ctx, maat_inst->logger); cleanup_update_status(mr_ctx->read_ctx, maat_inst->logger); redlock_unlock(mr_ctx->read_ctx, mr_expire_lock); @@ -1464,7 +1463,8 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, } if (NULL == mr_ctx->read_ctx || mr_ctx->read_ctx->err) { - if (time(NULL) - mr_ctx->last_reconnect_time < MAAT_REDIS_RECONNECT_INTERVAL_S) { + if ((time(NULL) - mr_ctx->last_reconnect_time) < + MAAT_REDIS_RECONNECT_INTERVAL_S) { return; } @@ -1476,9 +1476,9 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, log_info(maat_inst->logger, MODULE_REDIS_MONITOR, "Reconnecting..."); mr_ctx->read_ctx = maat_connect_redis(mr_ctx->redis_ip, - mr_ctx->redis_port, - mr_ctx->redis_db, - maat_inst->logger); + mr_ctx->redis_port, + mr_ctx->redis_db, + maat_inst->logger); if (NULL == mr_ctx->read_ctx) { return; } else { @@ -1491,11 +1491,11 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, int update_type = MAAT_UPDATE_TYPE_INC; int rule_num = maat_get_rm_key_list(mr_ctx->read_ctx, version, - maat_inst->load_specific_version, - &new_version, maat_inst->tbl_mgr, - &rule_list, &update_type, - maat_inst->opts.cumulative_update_off, - maat_inst->logger); + maat_inst->load_specific_version, + &new_version, maat_inst->tbl_mgr, + &rule_list, &update_type, + maat_inst->opts.cumulative_update_off, + maat_inst->logger); //redis communication error if (rule_num < 0) { redisFree(mr_ctx->read_ctx); @@ -1538,7 +1538,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, maat_inst, maat_inst->opts.foreign_cont_dir); if (ret > 0) { maat_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0, - maat_inst->logger); + maat_inst->logger); } } @@ -1562,10 +1562,10 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, if (rule_list[i].op == MAAT_OP_DEL) { valid_column = table_manager_get_valid_column(maat_inst->tbl_mgr, table_id); - ret = invalidate_line(rule_list[i].table_line, valid_column); + ret = validate_line(rule_list[i].table_line, valid_column); if (ret < 0) { log_error(maat_inst->logger, MODULE_REDIS_MONITOR, - "[%s:%d] Invalidate line failed, invaid format %s", + "[%s:%d] Validate line failed, invaid format %s", __FUNCTION__, __LINE__, rule_list[i].table_line); continue; } diff --git a/src/maat_virtual.c b/src/maat_virtual.c index 464c407..9e9494e 100644 --- a/src/maat_virtual.c +++ b/src/maat_virtual.c @@ -13,7 +13,6 @@ #include "maat_kv.h" #include "maat_utils.h" #include "log/log.h" -#include "maat_virtual.h" #include "maat_rule.h" #include "maat_table.h" diff --git a/test/maat_json.json b/test/maat_json.json index 08109ff..1beb1db 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -3225,7 +3225,227 @@ ] } ] - } + }, + { + "compile_id": 214, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "virtual_table": "HTTP_DUMMY", + "group_name": "NOTClauseAndExcludeGroup214_1", + "not_flag": 0, + "clause_index": 0, + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "keywords-dummy-214-1", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "virtual_table": "HTTP_DUMMY", + "group_name": "NOTClauseAndExcludeGroup214_2", + "not_flag": 1, + "clause_index": 1, + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "keywords-dummy-214-2", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "virtual_table": "HTTP_DUMMY", + "group_name": "NOTClauseAndExcludeGroup214_3", + "not_flag": 1, + "clause_index": 2, + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "keywords-dummy-214-3", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "virtual_table": "HTTP_DUMMY", + "group_name": "NOTClauseAndExcludeGroup214_4", + "not_flag": 1, + "clause_index": 3, + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "keywords-dummy-214-4", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "virtual_table": "HTTP_DUMMY", + "group_name": "NOTClauseAndExcludeGroup214_5", + "not_flag": 1, + "clause_index": 4, + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "keywords-dummy-214-5", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "virtual_table": "HTTP_DUMMY", + "group_name": "NOTClauseAndExcludeGroup214_6", + "not_flag": 1, + "clause_index": 5, + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "keywords-dummy-214-6", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "virtual_table": "HTTP_DUMMY", + "group_name": "NOTClauseAndExcludeGroup214_7", + "not_flag": 1, + "clause_index": 6, + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "keywords-dummy-214-7", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "virtual_table": "HTTP_DUMMY", + "group_name": "NOTClauseAndExcludeGroup214_8", + "not_flag": 1, + "clause_index": 7, + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "keywords-dummy-214-8", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 215, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "virtual_table": "HTTP_DUMMY", + "group_name": "NOTClauseAndExcludeGroup215_1", + "not_flag": 0, + "clause_index": 0, + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "keywords-dummy-215-1", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "virtual_table": "HTTP_DUMMY", + "group_name": "NOTClauseAndExcludeGroup215_2", + "not_flag": 1, + "clause_index": 1, + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "keywords-dummy-215-2", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "virtual_table": "HTTP_DUMMY", + "group_name": "NOTClauseAndExcludeGroup215_3", + "not_flag": 1, + "clause_index": 2, + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "keywords-dummy-215-3", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + } ], "plugin_table": [ { diff --git a/test/table_info.conf b/test/table_info.conf index 29c8b72..8829dac 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -633,5 +633,11 @@ "table_name":"HTTP_RESPONSE_KEYWORDS_8", "table_type":"virtual", "physical_table": "KEYWORDS_TABLE" + }, + { + "table_id":56, + "table_name":"HTTP_DUMMY", + "table_type":"virtual", + "physical_table": "KEYWORDS_TABLE" } ]
\ No newline at end of file |
