summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorliuwentan <[email protected]>2023-09-12 16:19:22 +0800
committerliuwentan <[email protected]>2023-10-09 12:06:29 +0800
commit929f0d0cdd50809b0e8889a568ff43e5f6ba94d2 (patch)
treee6003ee2c9592f2fee158c4bd1ae444b0da21228
parent68825a78f7256d9a889affe5e21123d91edfe83c (diff)
[PATCH]Each virtual table supports up to 8 NOT clauses
-rw-r--r--src/maat_compile.c91
-rw-r--r--src/maat_redis_monitor.c68
-rw-r--r--src/maat_virtual.c1
-rw-r--r--test/maat_json.json222
-rw-r--r--test/table_info.conf6
5 files changed, 341 insertions, 47 deletions
diff --git a/src/maat_compile.c b/src/maat_compile.c
index 9e28c17..1b858d0 100644
--- a/src/maat_compile.c
+++ b/src/maat_compile.c
@@ -83,6 +83,12 @@ struct literal_clause {
UT_hash_handle hh;
};
+struct vtable_not_clause {
+ int vtable_id;
+ int not_clause_num;
+ UT_hash_handle hh;
+};
+
/* compile_runtime and group2compile_runtime share compile_hash_map */
struct compile_runtime {
struct bool_matcher *bm;
@@ -100,10 +106,11 @@ struct compile_runtime {
};
struct group2compile_runtime {
- long long not_clause_num;
+ long long not_clause_cnt;
long long rule_num;
long long update_err_cnt;
struct compile_runtime *ref_compile_rt;
+ struct vtable_not_clause *not_clause_hash;
};
struct maat_clause {
@@ -683,6 +690,7 @@ void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num,
}
struct group2compile_runtime *g2c_rt = ALLOC(struct group2compile_runtime, 1);
+ g2c_rt->not_clause_hash = NULL;
return g2c_rt;
}
@@ -700,6 +708,16 @@ void group2compile_runtime_free(void *g2c_runtime)
return;
}
+ struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
+ if (g2c_rt->not_clause_hash != NULL) {
+ struct vtable_not_clause *not_clause = NULL, *tmp_not_clause = NULL;
+ HASH_ITER(hh, g2c_rt->not_clause_hash, not_clause, tmp_not_clause) {
+ HASH_DEL(g2c_rt->not_clause_hash, not_clause);
+ FREE(not_clause);
+ }
+ }
+ assert(g2c_rt->not_clause_hash == NULL);
+
FREE(g2c_runtime);
}
@@ -1901,7 +1919,52 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
return 0;
}
-#define MAX_NOT_CLAUSE_NUM 16
+
+#define MAX_NOT_CLAUSE_NUM 8
+int validate_vtable_not_clause(struct group2compile_runtime *g2c_rt,
+ struct table_manager *tbl_mgr, int vtable_id,
+ int is_valid, struct log_handle *logger)
+{
+ enum table_type table_type = table_manager_get_table_type(tbl_mgr, vtable_id);
+ if (table_type != TABLE_TYPE_VIRTUAL) {
+ log_fatal(logger, MODULE_COMPILE,
+ "[%s:%d]table(table_id:%d) is not virtual table, can't own NOT clause.",
+ __FUNCTION__, __LINE__, vtable_id);
+ return -1;
+ }
+
+ struct vtable_not_clause *not_clause = NULL;
+ HASH_FIND_INT(g2c_rt->not_clause_hash, &vtable_id, not_clause);
+
+ if (0 == is_valid) {
+ //delete
+ if (NULL == not_clause || 0 == not_clause->not_clause_num) {
+ return 0;
+ } else {
+ not_clause->not_clause_num--;
+ }
+ } else {
+ //add
+ if (NULL == not_clause) {
+ not_clause = ALLOC(struct vtable_not_clause, 1);
+ not_clause->vtable_id = vtable_id;
+ not_clause->not_clause_num++;
+ HASH_ADD_INT(g2c_rt->not_clause_hash, vtable_id, not_clause);
+ } else {
+ if (not_clause->not_clause_num >= MAX_NOT_CLAUSE_NUM) {
+ const char *table_name = table_manager_get_table_name(tbl_mgr, vtable_id);
+ log_fatal(logger, MODULE_COMPILE,
+ "[%s:%d]virtual table:<%s> NOT clause num exceed maximum:%d",
+ __FUNCTION__, __LINE__, table_name, MAX_NOT_CLAUSE_NUM);
+ return -1;
+ }
+ not_clause->not_clause_num++;
+ }
+ }
+
+ return 0;
+}
+
int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
const char *table_name, const char *line,
int valid_column)
@@ -1932,13 +1995,24 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
return -1;
}
+ if (1 == g2c_item->not_flag) {
+ ret = validate_vtable_not_clause(g2c_rt, schema->ref_tbl_mgr, g2c_item->vtable_id,
+ is_valid, compile_rt->logger);
+ if (ret < 0) {
+ log_fatal(compile_rt->logger, MODULE_COMPILE,
+ "[%s:%d]validate NOT clause failed, abandon config:%s",
+ __FUNCTION__, __LINE__, line);
+ goto next;
+ }
+ }
+
if (0 == is_valid) {
//delete
ret = maat_remove_group_from_compile(compile_rt->cfg_hash, g2c_item,
compile_rt->logger);
if (0 == ret) {
if (g2c_item->not_flag) {
- g2c_rt->not_clause_num--;
+ g2c_rt->not_clause_cnt--;
}
g2c_rt->rule_num--;
} else {
@@ -1950,20 +2024,15 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
compile_rt->logger);
if (0 == ret) {
if (g2c_item->not_flag) {
- g2c_rt->not_clause_num++;
+ g2c_rt->not_clause_cnt++;
}
g2c_rt->rule_num++;
} else {
g2c_rt->update_err_cnt++;
}
}
-
- if (g2c_rt->not_clause_num > MAX_NOT_CLAUSE_NUM) {
- log_fatal(compile_rt->logger, MODULE_COMPILE,
- "[%s:%d] NOT clause num:%lld exceed maximum:%d which may affect scanning performance",
- __FUNCTION__, __LINE__, g2c_rt->not_clause_num, MAX_NOT_CLAUSE_NUM);
- }
+next:
group2compile_item_free(g2c_item);
return ret;
}
@@ -1975,7 +2044,7 @@ long long group2compile_runtime_not_clause_count(void *g2c_runtime)
}
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
- return g2c_rt->not_clause_num;
+ return g2c_rt->not_clause_cnt;
}
long long group2compile_runtime_rule_count(void *g2c_runtime)
diff --git a/src/maat_redis_monitor.c b/src/maat_redis_monitor.c
index fb40936..d446707 100644
--- a/src/maat_redis_monitor.c
+++ b/src/maat_redis_monitor.c
@@ -43,11 +43,11 @@ const char *foreign_key_prefix = "__FILE_";
const char *mr_op_str[] = {"DEL", "ADD", "RENEW_TIMEOUT"};
-#define POSSIBLE_REDIS_REPLY_SIZE 2
+#define REDIS_REPLY_SIZE 2
struct expected_reply {
int s_rule_seq;
int possible_reply_num;
- redisReply possible_replies[POSSIBLE_REDIS_REPLY_SIZE];
+ redisReply possible_replies[REDIS_REPLY_SIZE];
};
static char *get_foreign_cont_filename(const char *table_name, long long rule_id,
@@ -867,7 +867,7 @@ static void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
log_error(logger, MODULE_REDIS_MONITOR,
"[%s:%d] Get %s,%lld foreign key %s content failed, redis server error",
__FUNCTION__, __LINE__,
- rule_list[track[i].rule_idx].table_name,
+ rule_list[track[i].rule_idx].table_name,
rule_list[track[i].rule_idx].rule_id,
rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key);
break;
@@ -886,15 +886,15 @@ static void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
FILE *fp = fopen(s_rule->f_keys[track[i].foreign_idx].filename, "w");
if (NULL == fp) {
log_error(logger, MODULE_REDIS_MONITOR,
- "[%s:%d] Write foreign content failed: fopen %s error",
- __FUNCTION__, __LINE__, s_rule->f_keys[track[i].foreign_idx].filename);
+ "[%s:%d] Write foreign content failed: fopen %s error", __FUNCTION__,
+ __LINE__, s_rule->f_keys[track[i].foreign_idx].filename);
} else {
fwrite(reply->str, 1, reply->len, fp);
fclose(fp);
fp = NULL;
if (1 == print_fn) {
- printf("[%s:%d] Written foreign content %s\n",
- __FUNCTION__, __LINE__, s_rule->f_keys[track[i].foreign_idx].filename);
+ printf("[%s:%d] Written foreign content %s\n", __FUNCTION__,
+ __LINE__, s_rule->f_keys[track[i].foreign_idx].filename);
}
}
}
@@ -920,7 +920,7 @@ void maat_get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
}
}
-static int invalidate_line(char *line, int column_seq)
+static int validate_line(char *line, int column_seq)
{
if (NULL == line || column_seq < 0) {
return -1;
@@ -957,7 +957,8 @@ void maat_rewrite_table_line_with_foreign(struct serial_rule *s_rule)
strncat(pos_rewrite_line, pos_origin_line, origin_column - pos_origin_line);
pos_rewrite_line += origin_column - pos_origin_line;
pos_origin_line = origin_column+origin_column_size;
- strncat(pos_rewrite_line, s_rule->f_keys[i].filename, strlen(s_rule->f_keys[i].filename));
+ strncat(pos_rewrite_line, s_rule->f_keys[i].filename,
+ strlen(s_rule->f_keys[i].filename));
pos_rewrite_line += strlen(s_rule->f_keys[i].filename);
}
@@ -971,7 +972,7 @@ static void expected_reply_add(struct expected_reply* expected, int s_rule_seq,
int type, long long integer)
{
int i = expected->possible_reply_num;
- assert(i < POSSIBLE_REDIS_REPLY_SIZE);
+ assert(i < REDIS_REPLY_SIZE);
expected->s_rule_seq = s_rule_seq;
expected->possible_replies[i].type = type;
expected->possible_replies[i].integer = integer;
@@ -1049,7 +1050,7 @@ const char* lua_exec_done=
"return maat_version;";
static redisReply* exec_serial_rule_end(redisContext *c, const char *transaction_list,
long long server_time, int renew_allowed,
- struct expected_reply *expect_reply, size_t *cnt)
+ struct expected_reply *expect_reply, size_t *cnt)
{
redisReply *data_reply = NULL;
@@ -1061,11 +1062,8 @@ static redisReply* exec_serial_rule_end(redisContext *c, const char *transaction
if (strlen(transaction_list) > 0) {
data_reply = maat_wrap_redis_command(c, "eval %s 4 MAAT_VERSION %s %s %s %lld",
- lua_exec_done,
- mr_status_sset,
- mr_version_sset,
- transaction_list,
- server_time);
+ lua_exec_done, mr_status_sset, mr_version_sset,
+ transaction_list, server_time);
freeReplyObject(data_reply);
data_reply = NULL;
expected_reply_add(expect_reply + *cnt, -1, REDIS_REPLY_INTEGER, 0);
@@ -1077,10 +1075,10 @@ static redisReply* exec_serial_rule_end(redisContext *c, const char *transaction
return data_reply;
}
-static void exec_serial_rule(redisContext *c, const char *transaction_list,
- struct serial_rule *s_rule, size_t rule_num,
- struct expected_reply *expect_reply, size_t *cnt,
- size_t offset, int renew_allowed)
+static void exec_serial_rule(redisContext *c, const char *transaction_list,
+ struct serial_rule *s_rule, size_t rule_num,
+ struct expected_reply *expect_reply, size_t *cnt,
+ size_t offset, int renew_allowed)
{
size_t i = 0;
size_t append_cmd_cnt = 0;
@@ -1235,7 +1233,7 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule,
char transaction_list[NAME_MAX * 2] = {0};
long long transaction_version = 0;
long long transaction_finished_version = 0;
- size_t max_multi_cmd_num = MAX_REDIS_OP_PER_SRULE * serial_rule_num + 2;// 2 for operation in exec_serial_rule_end()
+ size_t max_multi_cmd_num = MAX_REDIS_OP_PER_SRULE * serial_rule_num + 2; // 2 for operation in exec_serial_rule_end()
struct expected_reply *expected_reply = ALLOC(struct expected_reply, max_multi_cmd_num);
for (i = 0; i < serial_rule_num; i++) {
@@ -1456,7 +1454,8 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
//authorized to write
if (mr_ctx->write_ctx != NULL && mr_ctx->write_ctx->err == 0) {
//For thread safe, deliberately use redis_read_ctx but not redis_write_ctx.
- if (1 == redlock_try_lock(mr_ctx->read_ctx, mr_expire_lock, mr_expire_lock_timeout_ms)) {
+ if (1 == redlock_try_lock(mr_ctx->read_ctx, mr_expire_lock,
+ mr_expire_lock_timeout_ms)) {
check_maat_expiration(mr_ctx->read_ctx, maat_inst->logger);
cleanup_update_status(mr_ctx->read_ctx, maat_inst->logger);
redlock_unlock(mr_ctx->read_ctx, mr_expire_lock);
@@ -1464,7 +1463,8 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
}
if (NULL == mr_ctx->read_ctx || mr_ctx->read_ctx->err) {
- if (time(NULL) - mr_ctx->last_reconnect_time < MAAT_REDIS_RECONNECT_INTERVAL_S) {
+ if ((time(NULL) - mr_ctx->last_reconnect_time) <
+ MAAT_REDIS_RECONNECT_INTERVAL_S) {
return;
}
@@ -1476,9 +1476,9 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
log_info(maat_inst->logger, MODULE_REDIS_MONITOR, "Reconnecting...");
mr_ctx->read_ctx = maat_connect_redis(mr_ctx->redis_ip,
- mr_ctx->redis_port,
- mr_ctx->redis_db,
- maat_inst->logger);
+ mr_ctx->redis_port,
+ mr_ctx->redis_db,
+ maat_inst->logger);
if (NULL == mr_ctx->read_ctx) {
return;
} else {
@@ -1491,11 +1491,11 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
int update_type = MAAT_UPDATE_TYPE_INC;
int rule_num = maat_get_rm_key_list(mr_ctx->read_ctx, version,
- maat_inst->load_specific_version,
- &new_version, maat_inst->tbl_mgr,
- &rule_list, &update_type,
- maat_inst->opts.cumulative_update_off,
- maat_inst->logger);
+ maat_inst->load_specific_version,
+ &new_version, maat_inst->tbl_mgr,
+ &rule_list, &update_type,
+ maat_inst->opts.cumulative_update_off,
+ maat_inst->logger);
//redis communication error
if (rule_num < 0) {
redisFree(mr_ctx->read_ctx);
@@ -1538,7 +1538,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
maat_inst, maat_inst->opts.foreign_cont_dir);
if (ret > 0) {
maat_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0,
- maat_inst->logger);
+ maat_inst->logger);
}
}
@@ -1562,10 +1562,10 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
if (rule_list[i].op == MAAT_OP_DEL) {
valid_column = table_manager_get_valid_column(maat_inst->tbl_mgr, table_id);
- ret = invalidate_line(rule_list[i].table_line, valid_column);
+ ret = validate_line(rule_list[i].table_line, valid_column);
if (ret < 0) {
log_error(maat_inst->logger, MODULE_REDIS_MONITOR,
- "[%s:%d] Invalidate line failed, invaid format %s",
+ "[%s:%d] Validate line failed, invaid format %s",
__FUNCTION__, __LINE__, rule_list[i].table_line);
continue;
}
diff --git a/src/maat_virtual.c b/src/maat_virtual.c
index 464c407..9e9494e 100644
--- a/src/maat_virtual.c
+++ b/src/maat_virtual.c
@@ -13,7 +13,6 @@
#include "maat_kv.h"
#include "maat_utils.h"
#include "log/log.h"
-#include "maat_virtual.h"
#include "maat_rule.h"
#include "maat_table.h"
diff --git a/test/maat_json.json b/test/maat_json.json
index 08109ff..1beb1db 100644
--- a/test/maat_json.json
+++ b/test/maat_json.json
@@ -3225,7 +3225,227 @@
]
}
]
- }
+ },
+ {
+ "compile_id": 214,
+ "service": 1,
+ "action": 1,
+ "do_blacklist": 1,
+ "do_log": 1,
+ "user_region": "anything",
+ "is_valid": "yes",
+ "groups": [
+ {
+ "virtual_table": "HTTP_DUMMY",
+ "group_name": "NOTClauseAndExcludeGroup214_1",
+ "not_flag": 0,
+ "clause_index": 0,
+ "regions": [
+ {
+ "table_name": "KEYWORDS_TABLE",
+ "table_type": "expr",
+ "table_content": {
+ "keywords": "keywords-dummy-214-1",
+ "expr_type": "none",
+ "match_method": "sub",
+ "format": "uncase plain"
+ }
+ }
+ ]
+ },
+ {
+ "virtual_table": "HTTP_DUMMY",
+ "group_name": "NOTClauseAndExcludeGroup214_2",
+ "not_flag": 1,
+ "clause_index": 1,
+ "regions": [
+ {
+ "table_name": "KEYWORDS_TABLE",
+ "table_type": "expr",
+ "table_content": {
+ "keywords": "keywords-dummy-214-2",
+ "expr_type": "none",
+ "match_method": "sub",
+ "format": "uncase plain"
+ }
+ }
+ ]
+ },
+ {
+ "virtual_table": "HTTP_DUMMY",
+ "group_name": "NOTClauseAndExcludeGroup214_3",
+ "not_flag": 1,
+ "clause_index": 2,
+ "regions": [
+ {
+ "table_name": "KEYWORDS_TABLE",
+ "table_type": "expr",
+ "table_content": {
+ "keywords": "keywords-dummy-214-3",
+ "expr_type": "none",
+ "match_method": "sub",
+ "format": "uncase plain"
+ }
+ }
+ ]
+ },
+ {
+ "virtual_table": "HTTP_DUMMY",
+ "group_name": "NOTClauseAndExcludeGroup214_4",
+ "not_flag": 1,
+ "clause_index": 3,
+ "regions": [
+ {
+ "table_name": "KEYWORDS_TABLE",
+ "table_type": "expr",
+ "table_content": {
+ "keywords": "keywords-dummy-214-4",
+ "expr_type": "none",
+ "match_method": "sub",
+ "format": "uncase plain"
+ }
+ }
+ ]
+ },
+ {
+ "virtual_table": "HTTP_DUMMY",
+ "group_name": "NOTClauseAndExcludeGroup214_5",
+ "not_flag": 1,
+ "clause_index": 4,
+ "regions": [
+ {
+ "table_name": "KEYWORDS_TABLE",
+ "table_type": "expr",
+ "table_content": {
+ "keywords": "keywords-dummy-214-5",
+ "expr_type": "none",
+ "match_method": "sub",
+ "format": "uncase plain"
+ }
+ }
+ ]
+ },
+ {
+ "virtual_table": "HTTP_DUMMY",
+ "group_name": "NOTClauseAndExcludeGroup214_6",
+ "not_flag": 1,
+ "clause_index": 5,
+ "regions": [
+ {
+ "table_name": "KEYWORDS_TABLE",
+ "table_type": "expr",
+ "table_content": {
+ "keywords": "keywords-dummy-214-6",
+ "expr_type": "none",
+ "match_method": "sub",
+ "format": "uncase plain"
+ }
+ }
+ ]
+ },
+ {
+ "virtual_table": "HTTP_DUMMY",
+ "group_name": "NOTClauseAndExcludeGroup214_7",
+ "not_flag": 1,
+ "clause_index": 6,
+ "regions": [
+ {
+ "table_name": "KEYWORDS_TABLE",
+ "table_type": "expr",
+ "table_content": {
+ "keywords": "keywords-dummy-214-7",
+ "expr_type": "none",
+ "match_method": "sub",
+ "format": "uncase plain"
+ }
+ }
+ ]
+ },
+ {
+ "virtual_table": "HTTP_DUMMY",
+ "group_name": "NOTClauseAndExcludeGroup214_8",
+ "not_flag": 1,
+ "clause_index": 7,
+ "regions": [
+ {
+ "table_name": "KEYWORDS_TABLE",
+ "table_type": "expr",
+ "table_content": {
+ "keywords": "keywords-dummy-214-8",
+ "expr_type": "none",
+ "match_method": "sub",
+ "format": "uncase plain"
+ }
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "compile_id": 215,
+ "service": 1,
+ "action": 1,
+ "do_blacklist": 1,
+ "do_log": 1,
+ "user_region": "anything",
+ "is_valid": "yes",
+ "groups": [
+ {
+ "virtual_table": "HTTP_DUMMY",
+ "group_name": "NOTClauseAndExcludeGroup215_1",
+ "not_flag": 0,
+ "clause_index": 0,
+ "regions": [
+ {
+ "table_name": "KEYWORDS_TABLE",
+ "table_type": "expr",
+ "table_content": {
+ "keywords": "keywords-dummy-215-1",
+ "expr_type": "none",
+ "match_method": "sub",
+ "format": "uncase plain"
+ }
+ }
+ ]
+ },
+ {
+ "virtual_table": "HTTP_DUMMY",
+ "group_name": "NOTClauseAndExcludeGroup215_2",
+ "not_flag": 1,
+ "clause_index": 1,
+ "regions": [
+ {
+ "table_name": "KEYWORDS_TABLE",
+ "table_type": "expr",
+ "table_content": {
+ "keywords": "keywords-dummy-215-2",
+ "expr_type": "none",
+ "match_method": "sub",
+ "format": "uncase plain"
+ }
+ }
+ ]
+ },
+ {
+ "virtual_table": "HTTP_DUMMY",
+ "group_name": "NOTClauseAndExcludeGroup215_3",
+ "not_flag": 1,
+ "clause_index": 2,
+ "regions": [
+ {
+ "table_name": "KEYWORDS_TABLE",
+ "table_type": "expr",
+ "table_content": {
+ "keywords": "keywords-dummy-215-3",
+ "expr_type": "none",
+ "match_method": "sub",
+ "format": "uncase plain"
+ }
+ }
+ ]
+ }
+ ]
+ }
],
"plugin_table": [
{
diff --git a/test/table_info.conf b/test/table_info.conf
index 29c8b72..8829dac 100644
--- a/test/table_info.conf
+++ b/test/table_info.conf
@@ -633,5 +633,11 @@
"table_name":"HTTP_RESPONSE_KEYWORDS_8",
"table_type":"virtual",
"physical_table": "KEYWORDS_TABLE"
+ },
+ {
+ "table_id":56,
+ "table_name":"HTTP_DUMMY",
+ "table_type":"virtual",
+ "physical_table": "KEYWORDS_TABLE"
}
] \ No newline at end of file