summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
author刘学利 <[email protected]>2020-09-28 16:53:40 +0800
committer刘学利 <[email protected]>2020-09-28 16:53:40 +0800
commit87acc1a6f68fe6b006314ddfcc85cd063f2cfe6f (patch)
treeb12833e1a15da1f082f10b23b101824e457942c8
parent34de556665305b47d4771199c18e0929df56521e (diff)
parentb29714d006ed1f1f01318683477b930d01aeecf4 (diff)
Merge branch 'feature-fqdn-plugin' into 'develop-version3'v3.1.0
Feature fqdn plugin See merge request MESA_framework/maat!53
-rw-r--r--inc/Maat_rule.h9
-rw-r--r--src/CMakeLists.txt2
-rw-r--r--src/entry/FQDN_engine.cpp352
-rw-r--r--src/entry/Maat_api.cpp157
-rw-r--r--src/entry/Maat_command.cpp51
-rw-r--r--src/entry/Maat_ex_data.cpp6
-rw-r--r--src/entry/Maat_rule.cpp71
-rw-r--r--src/entry/Maat_table.cpp566
-rw-r--r--src/entry/Maat_table_runtime.cpp261
-rw-r--r--src/inc_internal/FQDN_engine.h69
-rw-r--r--src/inc_internal/Maat_ex_data.h3
-rw-r--r--src/inc_internal/Maat_rule_internal.h12
-rw-r--r--src/inc_internal/Maat_table.h66
-rw-r--r--src/inc_internal/Maat_table_runtime.h19
-rw-r--r--test/maat_json.json101
-rw-r--r--test/table_info.conf4
-rw-r--r--test/test_maatframe.cpp196
17 files changed, 1579 insertions, 366 deletions
diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h
index f825658..d22aabd 100644
--- a/inc/Maat_rule.h
+++ b/inc/Maat_rule.h
@@ -319,6 +319,15 @@ struct ip_address
int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data);
+int Maat_fqdn_plugin_EX_register(Maat_feather_t feather, int table_id,
+ Maat_plugin_EX_new_func_t* new_func,
+ Maat_plugin_EX_free_func_t* free_func,
+ Maat_plugin_EX_dup_func_t* dup_func,
+ long argl, void *argp);
+
+//Return order: Longger suffix first, then fqdn with bigger index first.
+int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data);
+
enum MAAT_RULE_OPT
{
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 7be2077..db37b65 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$
message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
add_definitions(-fPIC)
-set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp)
+set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp entry/FQDN_engine.cpp)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/)
include_directories(/opt/MESA/include/MESA/)
diff --git a/src/entry/FQDN_engine.cpp b/src/entry/FQDN_engine.cpp
new file mode 100644
index 0000000..d402f66
--- /dev/null
+++ b/src/entry/FQDN_engine.cpp
@@ -0,0 +1,352 @@
+/*
+ *
+ * Copyright (c) 2020
+ * String Algorithms Research Group
+ * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS)
+ * National Engineering Laboratory for Information Security Technologies (NELIST)
+ * All rights reserved
+ *
+ * Written by: LIU YANBING ([email protected])
+ * Last modification: 2020-09-01
+ *
+ * This code is the exclusive and proprietary property of IIE-CAS and NELIST.
+ * Usage for direct or indirect commercial advantage is not allowed without
+ * written permission from the authors.
+ *
+ */
+
+#include "FQDN_engine.h"
+#include <math.h>
+#include <ctype.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+/*************************************************************************************/
+
+//#include <nmmintrin.h>
+//#define popcnt_u64 _mm_popcnt_u64
+//Use gcc builtin function to replace SSE4.2 instruction for portability
+
+#ifdef _MSC_VER
+#include <intrin.h>
+#define popcnt_u64 __popcnt
+#else
+#define popcnt_u64 __builtin_popcountl
+#endif
+
+#define FOR(i, n) for(int i=0, _n=(int)(n); i<_n; i++)
+
+struct packedRT_t
+{
+ unsigned long long bitmap[4];
+ unsigned int A;
+ unsigned char B[4];
+};
+
+void * aligned_malloc(size_t size, size_t align)
+{
+ void * malloc_ptr;
+ void * aligned_ptr;
+
+ /* Error if align is not a power of two. */
+ if (align & (align - 1))
+ {
+ return ((void*) 0);
+ }
+
+ if (align==0 || size == 0)
+ {
+ return ((void *) 0);
+ }
+
+ malloc_ptr = malloc (sizeof(void *) + align - 1 + size);
+ if (!malloc_ptr)
+ {
+ return ((void *) 0);
+ }
+
+ aligned_ptr = (void *) (((size_t)malloc_ptr + sizeof(void *) + align-1) & ~(align-1));
+
+ ((void **) aligned_ptr) [-1] = malloc_ptr;
+
+ return aligned_ptr;
+}
+
+void aligned_free(void * aligned_ptr)
+{
+ if (aligned_ptr)
+ {
+ free (((void **) aligned_ptr) [-1]);
+ }
+}
+
+/*************************************************************************************/
+struct domain_impl_t
+{
+ unsigned int id;
+ int suf_match;
+ unsigned int len;
+ unsigned long long hash; /*��64λ��ϣֵΨһ��ʾһ������*/
+ domain_impl_t * next;
+ void * utag;
+};
+
+class CHashTrieFQDN
+{
+public:
+ CHashTrieFQDN();
+ ~CHashTrieFQDN();
+
+ int initialize(const struct FQDN_rule * rules, size_t n_rule);
+ int search(const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result);
+
+protected:
+ unsigned int rank(unsigned int h);
+
+protected:
+ unsigned int m_num;
+ domain_impl_t * m_domains;
+ unsigned int m_H;
+ unsigned int m_max_pat_len;
+ unsigned long long * m_B;
+ packedRT_t * m_RT;
+ domain_impl_t ** m_matched;
+ unsigned char m_case_tab[256];
+};
+
+
+const unsigned long long ONE=1;
+
+#define is_bit_set(tbl, off) ( tbl[off>>6] & ( ONE<<(off&63) ) )
+#define set_bit(tbl, off) ( tbl[off>>6] |= ( ONE<<(off&63) ) )
+
+const unsigned long long A=6364136223846793005LL;
+
+CHashTrieFQDN::CHashTrieFQDN()
+{
+ m_num=0;
+ m_domains=NULL;
+ m_B=NULL;
+ m_RT=NULL;
+ m_matched=NULL;
+ FOR(c, 256) m_case_tab[c]=tolower(c);
+}
+
+CHashTrieFQDN::~CHashTrieFQDN()
+{
+ if(m_domains!=NULL)
+ {
+ delete [] m_domains;
+ }
+
+ if(m_B!=NULL)
+ {
+ delete [] m_B;
+ }
+
+ if(m_RT!=NULL)
+ {
+ aligned_free(m_RT);
+ }
+
+ if(m_matched!=NULL)
+ {
+ delete [] m_matched;
+ }
+}
+
+int CHashTrieFQDN::initialize(const struct FQDN_rule * rules, size_t n_rule)
+{
+ long long mem_bytes=0;
+
+ if(n_rule==0) return -1;
+ m_num=n_rule;
+ m_domains=new domain_impl_t[m_num];
+ mem_bytes+=m_num*sizeof(domain_impl_t);
+
+ unsigned int N=m_num;
+ m_max_pat_len=0;
+
+ FOR(k, m_num)
+ {
+ m_domains[k].id =rules[k].id;
+ m_domains[k].suf_match=rules[k].is_suffix_match;
+ m_domains[k].len=rules[k].len;
+ m_domains[k].next=NULL;
+ m_domains[k].utag=rules[k].user_tag;
+
+ FOR(j, rules[k].len)
+ {
+ if(rules[k].FQDN[j]=='.') ++N;
+ }
+
+ if(m_max_pat_len<rules[k].len)
+ {
+ m_max_pat_len=rules[k].len;
+ }
+ }
+
+ unsigned int w=(int)(log10((double)N)/log10(2.0)+0.5)+4;
+ if(w<8) w=8;
+ m_H=(1U<<w);
+
+ m_B=new unsigned long long[m_H>>6];
+ mem_bytes+=(m_H>>6)*sizeof(unsigned long long);
+ FOR(i, (m_H>>6)) m_B[i]=0;
+
+ m_RT=(packedRT_t *)aligned_malloc(sizeof(packedRT_t)*((m_H>>8)+1), 64);
+ mem_bytes+=((m_H>>8)+1)*sizeof(packedRT_t);
+ FOR(i, (m_H>>8))
+ {
+ FOR(j, 4) m_RT[i].bitmap[j]=0;
+ }
+
+ FOR(k, m_num)
+ {
+ const unsigned char * pb=(const unsigned char *)rules[k].FQDN;
+ m_domains[k].hash=0;
+
+ for(int j=rules[k].len-1; j>=-1; --j)
+ {
+ if(j==-1 || pb[j]=='.')
+ {
+ unsigned int h=m_domains[k].hash&(m_H-1);
+ set_bit(m_B, h);
+ if(j==-1)
+ {
+ int q=h&255;
+ m_RT[h>>8].bitmap[q>>6]|=(ONE<<(q&63));
+ }
+ }
+ else
+ {
+ m_domains[k].hash=A*m_domains[k].hash+m_case_tab[pb[j]];
+ }
+ }
+ }
+
+ m_RT[0].A=0;
+ FOR(i, (m_H>>8))
+ {
+ m_RT[i].B[0]=0;
+ m_RT[i].B[1]= popcnt_u64(m_RT[i].bitmap[0]);
+ m_RT[i].B[2]= m_RT[i].B[1]+popcnt_u64(m_RT[i].bitmap[1]);
+ m_RT[i].B[3]= m_RT[i].B[2]+popcnt_u64(m_RT[i].bitmap[2]);
+ m_RT[i+1].A=m_RT[i].A+m_RT[i].B[3]+popcnt_u64(m_RT[i].bitmap[3]);
+ }
+
+ int tn=m_RT[m_H>>8].A;
+
+ m_matched=new domain_impl_t *[tn];
+ mem_bytes+=tn*sizeof(domain_impl_t *);
+ FOR(i, tn) m_matched[i]=NULL;
+
+ FOR(k, m_num)
+ {
+ unsigned int h=m_domains[k].hash&(m_H-1);
+ unsigned idx=rank(h);
+ m_domains[k].next=m_matched[idx];
+ m_matched[idx]=&(m_domains[k]);
+ }
+
+// printf("mem_bytes=%u(MB)\n", mem_bytes/(1U<<20));
+
+ return 1;
+}
+
+unsigned int CHashTrieFQDN::rank(unsigned int h)
+{
+ int p=(h>>8);
+ int r=((h&255)>>6);
+ int s=(h&63);
+ unsigned long long e=m_RT[p].bitmap[r]&((ONE<<s)-1);
+
+ return m_RT[p].A+m_RT[p].B[r]+popcnt_u64(e);
+}
+
+int CHashTrieFQDN::search(const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result)
+{
+ if(m_num==0 || FQDN_len==0 || FQDN==NULL || n_result==0) return -1;
+
+ size_t match_num=0;
+ const unsigned char * pb=(unsigned char *)FQDN;
+ unsigned long long HASH[16]; /*������������������16*/
+ unsigned int P[16];
+ int t=0;
+ unsigned long long hash=0;
+ for(int j=FQDN_len-1; j>=-1; --j)
+ {
+ if(j==-1 || pb[j]=='.')
+ {
+ unsigned int h=hash&(m_H-1);
+ if(is_bit_set(m_B, h)==0) break;
+ HASH[t]=hash;
+ P[t]=j+1;
+ ++t;
+ }
+ else if(j+m_max_pat_len<FQDN_len) break;
+ else
+ {
+ hash=A*hash+m_case_tab[pb[j]];
+ }
+ }
+
+ for(--t; t>=0; t--)
+ {
+ unsigned int h=HASH[t]&(m_H-1);
+ int q=h&255;
+
+ if(m_RT[h>>8].bitmap[q>>6]&(ONE<<(q&63)))
+ {
+ unsigned idx=rank(h);
+
+ for(domain_impl_t * pt=m_matched[idx]; pt!=NULL; pt=pt->next)
+ {
+ if(P[t]!=0 && pt->suf_match==0) continue;
+ if(pt->len+P[t]==FQDN_len && pt->hash==HASH[t])
+ {
+ //if(match_num>0 && P[t]!=results[match_num-1].offset) return match_num;
+ results[match_num].id=pt->id;
+ results[match_num].offset=P[t];
+ results[match_num].user_tag=pt->utag;
+ ++match_num;
+ if(match_num==n_result) return match_num;
+ }
+ }
+ }
+ }
+
+ return match_num;
+}
+
+/*************************************************************************************/
+struct FQDN_engine
+{
+ CHashTrieFQDN ht;
+};
+
+struct FQDN_engine * FQDN_engine_new(const struct FQDN_rule * rules, size_t n_rule)
+{
+ struct FQDN_engine * instance=new struct FQDN_engine;
+ if(instance->ht.initialize(rules, n_rule)<0)
+ {
+ delete instance;
+ return NULL;
+ }
+ else
+ {
+ return instance;
+ }
+}
+
+int FQDN_engine_search(struct FQDN_engine * instance, const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result)
+{
+ if(instance==NULL) return -1;
+ return instance->ht.search(FQDN, FQDN_len, results, n_result);
+}
+
+void FQDN_engine_free(struct FQDN_engine * instance)
+{
+ if(instance!=NULL) delete instance;
+}
diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp
index c8c5f8b..95088c0 100644
--- a/src/entry/Maat_api.cpp
+++ b/src/entry/Maat_api.cpp
@@ -1125,13 +1125,30 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id
struct Maat_table_schema *table_desc=NULL;
struct Maat_table_runtime *table_rt=NULL;
MAAT_RULE_EX_DATA exdata=NULL;
+ struct timespec start,end;
+ if(_feather->perf_on==1)
+ {
+ clock_gettime(CLOCK_MONOTONIC,&start);
+ }
+
if(_feather->scanner==NULL)
{
return NULL;
}
- table_desc=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
+ table_desc=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL);
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
exdata=Maat_table_runtime_plugin_get_ex_data(table_rt, table_desc, key);
+
+ if(_feather->perf_on==1)
+ {
+ clock_gettime(CLOCK_MONOTONIC,&end);
+ maat_stat_table(table_rt, 0, &start, &end, 0);
+ }
+ else
+ {
+ maat_stat_table(table_rt, 0, NULL, NULL, 0);
+ }
+
return exdata;
}
int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id,
@@ -1163,6 +1180,35 @@ int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id,
return 0;
}
+int Maat_fqdn_plugin_EX_register(Maat_feather_t feather, int table_id,
+ Maat_plugin_EX_new_func_t* new_func,
+ Maat_plugin_EX_free_func_t* free_func,
+ Maat_plugin_EX_dup_func_t* dup_func,
+ long argl, void *argp)
+{
+ struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
+ int ret=-1;
+ struct Maat_table_schema *table_schema=Maat_table_get_by_id_raw(_feather->table_mgr, table_id);
+ pthread_mutex_lock(&(_feather->background_update_mutex));
+ ret=Maat_table_fqdn_plugin_EX_data_schema_set(table_schema, new_func, free_func, dup_func, NULL, argl, argp, _feather->logger);
+
+ if(ret<0)
+ {
+ pthread_mutex_unlock(&(_feather->background_update_mutex));
+ return -1;
+ }
+
+ struct Maat_table_runtime* table_rt=NULL;
+ if(_feather->scanner!=NULL)
+ {
+ table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
+ Maat_table_runtime_fqdn_plugin_commit_ex_schema(table_rt, table_schema, _feather->logger);
+ }
+ pthread_mutex_unlock(&(_feather->background_update_mutex));
+
+ return 0;
+
+}
int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data)
{
@@ -1170,11 +1216,18 @@ int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struc
struct Maat_table_schema *table_schema=NULL;
struct Maat_table_runtime *table_rt=NULL;
int n_get=0;
+
+ struct timespec start,end;
+ if(_feather->perf_on==1)
+ {
+ clock_gettime(CLOCK_MONOTONIC,&start);
+ }
if(_feather->scanner==NULL)
{
return 0;
}
- table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
+
+ table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_IP_PLUGIN, NULL);
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
if(table_rt->table_type!=TABLE_TYPE_IP_PLUGIN)
{
@@ -1190,9 +1243,57 @@ int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struc
ipv6_ntoh(ip_data.ipv6);
}
n_get=Maat_table_runtime_ip_plugin_get_N_ex_data(table_rt, table_schema, &ip_data, ex_data_array, n_ex_data);
+
+ if(_feather->perf_on==1)
+ {
+ clock_gettime(CLOCK_MONOTONIC,&end);
+ maat_stat_table(table_rt, 0, &start, &end, 0);
+ }
+ else
+ {
+ maat_stat_table(table_rt, 0, NULL, NULL, 0);
+ }
+
return n_get;
}
+int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data)
+{
+ struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
+ struct Maat_table_schema *table_schema=NULL;
+ struct Maat_table_runtime *table_rt=NULL;
+ int n_get=0;
+
+ struct timespec start,end;
+ if(_feather->perf_on==1)
+ {
+ clock_gettime(CLOCK_MONOTONIC,&start);
+ }
+ if(_feather->scanner==NULL)
+ {
+ return 0;
+ }
+ table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_FQDN_PLUGIN, NULL);
+ table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
+ if(table_rt->table_type!=TABLE_TYPE_FQDN_PLUGIN)
+ {
+ return -1;
+ }
+ n_get=Maat_table_runtime_fqdn_plugin_get_N_ex_data(table_rt, table_schema, fqdn, ex_data_array, n_ex_data);
+
+
+ if(_feather->perf_on==1)
+ {
+ clock_gettime(CLOCK_MONOTONIC,&end);
+ maat_stat_table(table_rt, 0, &start, &end, 0);
+ }
+ else
+ {
+ maat_stat_table(table_rt, 0, NULL, NULL, 0);
+ }
+
+ return n_get;
+}
int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
,enum MAAT_CHARSET charset,const char* data,int data_len
@@ -1227,7 +1328,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
return 0;
}
- p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id);
+ p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -1365,7 +1466,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
_mid->scan_cnt++;
int virtual_table_id=0;
- p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_INTERVAL, &virtual_table_id);
+ p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_INTERVAL, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -1467,7 +1568,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
_mid->scan_cnt++;
int virtual_table_id=0;
- p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_SIMILARITY, &virtual_table_id);
+ p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -1531,7 +1632,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
return compile_ret;
}
-static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, unsigned short int proto, enum MAAT_TABLE_CHILD_TYPE child_type, int table_id)
+static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, unsigned short int proto, enum MAAT_TABLE_COMPONENT_TYPE child_type, int table_id)
{
memset(scan_data, 0, sizeof(scan_data_t));
scan_data->sub_type=make_sub_type(table_id, CHARSET_NONE, 0);
@@ -1542,15 +1643,15 @@ static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, u
scan_data->ipv4_data.proto=proto;
switch(child_type)
{
- case CHILD_TABLE_TYPE_SOURCE_IP:
+ case COMPONENT_TABLE_TYPE_SOURCE_IP:
scan_data->ipv4_data.saddr=ntohl(addr->v4->saddr);
scan_data->ipv4_data.sport=ntohs(addr->v4->source);
break;
- case CHILD_TABLE_TYPE_DESTINATION_IP:
+ case COMPONENT_TABLE_TYPE_DESTINATION_IP:
scan_data->ipv4_data.saddr=ntohl(addr->v4->daddr);
scan_data->ipv4_data.sport=ntohs(addr->v4->dest);
break;
- case CHILD_TABLE_TYPE_SESSION:
+ case COMPONENT_TABLE_TYPE_SESSION:
scan_data->ipv4_data.saddr=ntohl(addr->v4->saddr);
scan_data->ipv4_data.sport=ntohs(addr->v4->source);
scan_data->ipv4_data.daddr=ntohl(addr->v4->daddr);
@@ -1566,17 +1667,17 @@ static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, u
scan_data->ipv6_data.proto=proto;
switch(child_type)
{
- case CHILD_TABLE_TYPE_SOURCE_IP:
+ case COMPONENT_TABLE_TYPE_SOURCE_IP:
memcpy(scan_data->ipv6_data.saddr, addr->v6->saddr, sizeof(scan_data->ipv6_data.saddr));
ipv6_ntoh(scan_data->ipv6_data.saddr);
scan_data->ipv6_data.sport=ntohs(addr->v6->source);
break;
- case CHILD_TABLE_TYPE_DESTINATION_IP:
+ case COMPONENT_TABLE_TYPE_DESTINATION_IP:
memcpy(scan_data->ipv6_data.saddr, addr->v6->daddr, sizeof(scan_data->ipv6_data.saddr));
ipv6_ntoh(scan_data->ipv6_data.saddr);
scan_data->ipv6_data.sport=ntohs(addr->v6->dest);
break;
- case CHILD_TABLE_TYPE_SESSION:
+ case COMPONENT_TABLE_TYPE_SESSION:
memcpy(scan_data->ipv6_data.saddr, addr->v6->saddr, sizeof(scan_data->ipv6_data.saddr));
ipv6_ntoh(scan_data->ipv6_data.saddr);
scan_data->ipv6_data.sport=ntohs(addr->v6->source);
@@ -1596,7 +1697,7 @@ static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, u
}
return 0;
}
-static int IP_composition_scan(const struct ipaddr* addr, unsigned short int proto, Maat_table_schema* parent_table, enum MAAT_TABLE_CHILD_TYPE child_type,
+static int IP_composition_scan(const struct ipaddr* addr, unsigned short int proto, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE child_type,
scan_result_t *region_result, unsigned int result_num,
int* virtual_table_id,
rule_scanner_t ip_scanner, struct Maat_table_manager* table_mgr, struct Maat_table_runtime_manager* table_rt_mgr,
@@ -1604,21 +1705,21 @@ static int IP_composition_scan(const struct ipaddr* addr, unsigned short int pro
{
int child_table_id=0;
- if(child_type==CHILD_TABLE_TYPE_NONE)
+ if(child_type==COMPONENT_TABLE_TYPE_NONE)
{
- child_table_id=parent_table->table_id;
- child_type=CHILD_TABLE_TYPE_SESSION;
+ child_table_id=parent_table_id;
+ child_type=COMPONENT_TABLE_TYPE_SESSION;
}
else
{
- child_table_id=Maat_table_get_child_id(parent_table, child_type);
+ child_table_id=Maat_table_get_child_id(table_mgr, parent_table_id, child_type);
}
if(child_table_id<0)
{
return 0;
}
- Maat_table_schema* real_table=Maat_table_get_scan_by_id(table_mgr, child_table_id, TABLE_TYPE_IP, virtual_table_id);
+ Maat_table_schema* real_table=Maat_table_get_scan_by_id(table_mgr, child_table_id, SCAN_TYPE_IP, virtual_table_id);
if(real_table==NULL)
{
return 0;
@@ -1659,7 +1760,6 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
int region_ret=0, compile_ret=0;
struct _OUTER_scan_status_t* _mid=NULL;
scan_result_t *region_result=NULL;
- Maat_table_schema* p_table=NULL;
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct Maat_scanner* my_scanner=NULL;
struct timespec start,end;
@@ -1670,8 +1770,9 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
_mid=grab_mid(mid, _feather, thread_num, 0);
_mid->scan_cnt++;
int virtual_table_id=0;
- p_table=Maat_table_get_by_id_raw(_feather->table_mgr, table_id);
- if(p_table==NULL)
+ enum MAAT_TABLE_TYPE table_type=TABLE_TYPE_INVALID;
+ table_type=Maat_table_get_type_by_id(_feather->table_mgr, table_id);
+ if(table_type==TABLE_TYPE_INVALID)
{
_feather->scan_err_cnt++;
return -1;
@@ -1690,12 +1791,12 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
INC_SCANNER_REF(my_scanner, thread_num);
- if(p_table->table_type==TABLE_TYPE_COMPOSITION)
+ if(table_type==TABLE_TYPE_COMPOSITION)
{
- enum MAAT_TABLE_CHILD_TYPE childs[3]={CHILD_TABLE_TYPE_SOURCE_IP, CHILD_TABLE_TYPE_DESTINATION_IP, CHILD_TABLE_TYPE_SESSION};
+ enum MAAT_TABLE_COMPONENT_TYPE childs[3]={COMPONENT_TABLE_TYPE_SOURCE_IP, COMPONENT_TABLE_TYPE_DESTINATION_IP, COMPONENT_TABLE_TYPE_SESSION};
for(int i=0; i<3; i++)
{
- region_ret=IP_composition_scan(addr, proto, p_table, childs[i],
+ region_ret=IP_composition_scan(addr, proto, table_id, childs[i],
region_result+region_hit_cnt, MAX_SCANNER_HIT_NUM-region_hit_cnt, &virtual_table_id,
my_scanner->region, _feather->table_mgr, _feather->scanner->table_rt_mgr, thread_num);
if(region_ret<0)
@@ -1713,7 +1814,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
}
else
{
- region_ret=IP_composition_scan(addr, proto, p_table, CHILD_TABLE_TYPE_NONE,
+ region_ret=IP_composition_scan(addr, proto, table_id, COMPONENT_TABLE_TYPE_NONE,
region_result+region_hit_cnt, MAX_SCANNER_HIT_NUM-region_hit_cnt, &virtual_table_id,
my_scanner->region, _feather->table_mgr, _feather->scanner->table_rt_mgr, thread_num);
if(region_ret<0)
@@ -1733,7 +1834,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
_mid=grab_mid(mid, _feather, thread_num, 1);
scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, region_hit_cnt,
_mid->is_last_region, virtual_table_id, _mid->scan_cnt);
- if(p_table->table_type==TABLE_TYPE_COMPOSITION)
+ if(table_type==TABLE_TYPE_COMPOSITION)
{
region_hit_wraper.virtual_table_ids=region_rslt_virtual_table_id;
}
@@ -1783,7 +1884,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
struct Maat_table_schema *p_table=NULL;
int virtual_table_id=0;
assert(thread_num<_feather->scan_thread_num);
- p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id);
+ p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
@@ -2071,7 +2172,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
sfh_instance_t * tmp_fuzzy_handle=NULL;
struct Maat_table_schema *p_table=NULL;
int virtual_table_id=0;
- p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_DIGEST, &virtual_table_id);
+ p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp
index fa48b80..02ce3ca 100644
--- a/src/entry/Maat_command.cpp
+++ b/src/entry/Maat_command.cpp
@@ -201,6 +201,7 @@ int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_
break;
case TABLE_TYPE_PLUGIN:
case TABLE_TYPE_IP_PLUGIN:
+ case TABLE_TYPE_FQDN_PLUGIN:
if(valid_column_seq<0)
{
return -1;
@@ -1624,7 +1625,8 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
int update_type=CM_UPDATE_TYPE_INC;
long long new_version=0;
enum MAAT_TABLE_TYPE table_type;
- struct Maat_table_schema* table_desc=NULL;
+ enum MAAT_SCAN_TYPE scan_type;
+ struct Maat_table_schema* table_schema=NULL;
void* logger=feather->logger;
if(mr_ctx->write_ctx!=NULL&&mr_ctx->write_ctx->err==0)//authorized to write
@@ -1713,26 +1715,14 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
table_type=Maat_table_get_type_by_id(feather->table_mgr, table_id);
if(rule_list[i].op==MAAT_OP_DEL)
{
- if(table_type==TABLE_TYPE_PLUGIN||table_type==TABLE_TYPE_IP_PLUGIN)
- {
- table_desc=Maat_table_get_scan_by_id(feather->table_mgr, table_id, table_type, NULL);
- if(table_type==TABLE_TYPE_PLUGIN)
- {
- valid_column=table_desc->plugin.valid_flag_column;
- }
- else
- {
- valid_column=table_desc->ip_plugin.valid_flag_column;
- }
- }
- else
- {
- valid_column=-1;
- }
+
+ scan_type=Maat_table_get_scan_type(table_type);
+ table_schema=Maat_table_get_scan_by_id(feather->table_mgr, table_id, scan_type, NULL);
+ valid_column=Maat_table_xx_plugin_table_get_valid_flag_column(table_schema);
ret=invalidate_line(rule_list[i].table_line, table_type, valid_column);
if(ret<0)
{
- MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor,"Invaid format %s ."
+ MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor,"Invalidate line failed, invaid format %s ."
,rule_list[i].table_line);
continue;
}
@@ -1918,22 +1908,17 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_
goto error_out;
}
int valid_flag_column=0;
- switch(p_table->table_type)
+
+ valid_flag_column=Maat_table_xx_plugin_table_get_valid_flag_column(p_table);
+ if(valid_flag_column<0)
{
- case TABLE_TYPE_PLUGIN:
- valid_flag_column=p_table->plugin.valid_flag_column;
- plugin_desc=&(p_table->plugin);
- break;
- case TABLE_TYPE_IP_PLUGIN:
- valid_flag_column=p_table->ip_plugin.valid_flag_column;
- break;
- default:
- MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command
- ,"Command set line id %d failed: table %s is not a plugin or ip_plugin table."
- , line_rule[i]->rule_id
- , line_rule[i]->table_name);
- ret=-1;
- goto error_out;
+ MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command
+ ,"Command set line id %d failed: table %s is not a plugin or ip_plugin table."
+ , line_rule[i]->rule_id
+ , line_rule[i]->table_name);
+ ret=-1;
+ goto error_out;
+
}
if(op==MAAT_OP_ADD)
diff --git a/src/entry/Maat_ex_data.cpp b/src/entry/Maat_ex_data.cpp
index 7cf2a77..a98645b 100644
--- a/src/entry/Maat_ex_data.cpp
+++ b/src/entry/Maat_ex_data.cpp
@@ -57,7 +57,7 @@ void cache_row_free(void*p)
}
UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free};
-struct EX_data_rt* EX_data_rt_new(int table_id, long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data))
+struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data))
{
struct EX_data_rt* p=ALLOC(struct EX_data_rt, 1);
p->hash_key2ex=NULL;
@@ -207,3 +207,7 @@ void* EX_data_container_get_user_data(struct EX_data_container* ex_container)
{
return ex_container->user_data;
}
+size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt)
+{
+ return HASH_COUNT(ex_rt->hash_key2ex);
+}
diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp
index e6c30e1..6b01597 100644
--- a/src/entry/Maat_rule.cpp
+++ b/src/entry/Maat_rule.cpp
@@ -34,7 +34,7 @@
#include "stream_fuzzy_hash.h"
#include "gram_index_engine.h"
-int MAAT_FRAME_VERSION_3_0_20200917=1;
+int MAAT_FRAME_VERSION_3_0_20200927=1;
int is_valid_table_name(const char* str)
{
@@ -1392,6 +1392,8 @@ int add_digest_rule(struct Maat_table_schema* table, struct db_digest_rule* db_r
scanner->gie_update_q_size++;
return 0;
}
+
+
int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id, int rule_type, struct Maat_scanner *maat_scanner, void* logger)
{
int i=0;
@@ -2163,7 +2165,6 @@ error_out:
digest_rule=NULL;
return;
}
-
void update_plugin_table(struct Maat_table_schema* table_schema, const char* row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger)
{
int ret=1, matched_tag=1;
@@ -2210,21 +2211,34 @@ void update_plugin_table(struct Maat_table_schema* table_schema, const char* row
}
Maat_table_runtime_plugin_new_row(table_rt, table_schema, row, logger);
}
-void update_ip_plugin_table(struct Maat_table_schema* table_schema, const char* table_row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger)
+void update_xx_plugin_table(struct Maat_table_schema* table_schema, const char* table_row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger)
{
int ret=1, matched_tag=1;
- struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin);
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_schema->table_id);
char* copy=NULL;
size_t accept_tag_offset=0, accept_tag_len=0;
- if(ip_plugin_schema->rule_tag_column>0&&n_tags>0)
+ int rule_tag_column=-1;
+ if(table_schema->table_type==TABLE_TYPE_IP_PLUGIN)
+ {
+ rule_tag_column=table_schema->ip_plugin.rule_tag_column;
+ }
+ else if(table_schema->table_type==TABLE_TYPE_FQDN_PLUGIN)
+ {
+ rule_tag_column=table_schema->fqdn_plugin.rule_tag_column;
+ }
+ else
{
- ret=Maat_helper_read_column(table_row, ip_plugin_schema->rule_tag_column, &accept_tag_offset, &accept_tag_len);
+ assert(0);
+ return;
+ }
+ if(rule_tag_column>0&&n_tags>0)
+ {
+ ret=Maat_helper_read_column(table_row, rule_tag_column, &accept_tag_offset, &accept_tag_len);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
- "update error, could not locate tag in column %d of plugin table_schema %s:%s",
- ip_plugin_schema->rule_tag_column,
+ "update error, could not locate tag in column %d of table %s:%s",
+ rule_tag_column,
table_schema->table_name[table_schema->updating_name],
table_row);
table_schema->udpate_err_cnt++;
@@ -2254,8 +2268,15 @@ void update_ip_plugin_table(struct Maat_table_schema* table_schema, const char*
return;
}
}
- Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger);
- scanner->ip_plugin_update_q_size++;
+ if(table_schema->table_type==TABLE_TYPE_IP_PLUGIN)
+ {
+ Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger);
+ }
+ else //TABLE_TYPE_FQDN_PLUGIN
+ {
+ Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, table_row, logger);
+ }
+ scanner->xx_plugin_update_q_size++;
return;
}
@@ -2263,9 +2284,9 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
{
MESA_htable_handle tmp_map=NULL;
struct Maat_table_runtime* table_rt=NULL;
- struct ip_matcher* old_ip_matcher=NULL;
int i=0, ret=0;
-
+ struct ip_matcher* old_ip_matcher=NULL;
+ struct FQDN_engine* old_fqdn_engine=NULL;
if(scanner->to_update_compile_cnt+scanner->to_update_group_cnt>0)
{
ret=Maat_hierarchy_rebuild(scanner->hier);
@@ -2299,8 +2320,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
switch(table_rt->table_type)
{
case TABLE_TYPE_DIGEST:
- case TABLE_TYPE_SIMILARITY:
-
+ case TABLE_TYPE_SIMILARITY:
ret=Maat_table_runtime_digest_batch_udpate(table_rt);
if(ret<0)
{
@@ -2308,7 +2328,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
"GIE_update error.");
}
break;
- case TABLE_TYPE_IP_PLUGIN:
+ case TABLE_TYPE_IP_PLUGIN:
ret=Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt);
if(ret)
{
@@ -2319,6 +2339,18 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
}
}
break;
+ case TABLE_TYPE_FQDN_PLUGIN:
+
+ ret=Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(table_rt);
+ if(ret)
+ {
+ old_fqdn_engine=Maat_table_runtime_apply_new_fqdn_engine(table_rt);
+ if(old_fqdn_engine)
+ {
+ Maat_garbage_bagging(scanner->ref_garbage_bin, old_fqdn_engine, (void (*)(void*))FQDN_engine_free);
+ }
+ }
+ break;
default:
break;
}
@@ -2335,7 +2367,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
scanner->gie_update_q_size=0;
scanner->to_update_group_cnt=0;
scanner->to_update_compile_cnt=0;
- scanner->ip_plugin_update_q_size=0;
+ scanner->xx_plugin_update_q_size=0;
return;
}
@@ -2400,7 +2432,7 @@ void maat_finish_cb(void* u_para)
feather->scanner->cfg_num=scanner_rule_num(feather->scanner);
feather->scanner->version=feather->maat_version;
expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q);
- feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size+feather->scanner->ip_plugin_update_q_size;
+ feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size+feather->scanner->xx_plugin_update_q_size;
if(time(NULL)-feather->scanner->last_update_time>=feather->effect_interval_ms/1000)
{
do_scanner_update(feather->scanner,
@@ -2480,8 +2512,9 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para)
update_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger);
break;
case TABLE_TYPE_IP_PLUGIN:
- update_ip_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger);
- break;
+ case TABLE_TYPE_FQDN_PLUGIN:
+ update_xx_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger);
+ break;
default:
break;
diff --git a/src/entry/Maat_table.cpp b/src/entry/Maat_table.cpp
index 2994445..552eb94 100644
--- a/src/entry/Maat_table.cpp
+++ b/src/entry/Maat_table.cpp
@@ -18,7 +18,45 @@ struct Maat_table_manager
MESA_htable_handle map_tablename2id;
int active_plugin_table_num;
int is_last_plugin_table_updating;
+ void* logger;
};
+enum MAAT_SCAN_TYPE Maat_table_get_scan_type(enum MAAT_TABLE_TYPE table_type)
+{
+ enum MAAT_SCAN_TYPE ret=SCAN_TYPE_INVALID;
+ switch(table_type)
+ {
+ case TABLE_TYPE_EXPR:
+ case TABLE_TYPE_EXPR_PLUS:
+ case TABLE_TYPE_SIMILARITY:
+ case TABLE_TYPE_DIGEST:
+ ret=SCAN_TYPE_STRING;
+ break;
+ case TABLE_TYPE_INTERVAL:
+ case TABLE_TYPE_INTERVAL_PLUS:
+ ret=SCAN_TYPE_INTERVAL;
+ break;
+ case TABLE_TYPE_IP:
+ case TABLE_TYPE_IP_PLUS:
+ case TABLE_TYPE_COMPOSITION:
+ ret=SCAN_TYPE_IP;
+ break;
+ case TABLE_TYPE_PLUGIN:
+ ret=SCAN_TYPE_PLUGIN;
+ break;
+ case TABLE_TYPE_IP_PLUGIN:
+ ret=SCAN_TYPE_IP;
+ break;
+ case TABLE_TYPE_FQDN_PLUGIN:
+ ret=SCAN_TYPE_FQDN_PLUGIN;
+ break;
+ case TABLE_TYPE_COMPILE:
+ ret=SCAN_TYPE_NONE;
+ break;
+ default:
+ break;
+ }
+ return ret;
+}
int read_expr_table_info(const char* line, struct Maat_table_schema* table, MESA_htable_handle string2int_map)
{
@@ -74,25 +112,7 @@ int read_expr_table_info(const char* line, struct Maat_table_schema* table, MESA
}
return 0;
}
-int read_virtual_table_schema(const char* line, struct Maat_table_schema* table, MESA_htable_handle string2int_map)
-{
- int ret=0;
- char table_type[16];
- ret=sscanf(line, "%d\t%s\t%s\t%s", &(table->table_id),
- table->table_name[0],
- table_type,
- table->virtual_table.real_table_name);
- if(ret!=4)
- {
- return -1;
- }
- ret=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type));
- if(ret<0)
- {
- return -1;
- }
- return 0;
-}
+
Maat_table_schema* table_info_new(void)
{
struct Maat_table_schema*p=ALLOC(struct Maat_table_schema, 1);
@@ -119,7 +139,83 @@ int _read_integer_arrary(char* string, int *array, int size)
}
#define COLUMN_PLUGIN_SCHEMA_JSON 4
#define COLUMN_IP_PLUGIN_SCHEMA_JSON 4
+#define COLUMN_FQDN_PLUGIN_SHCEMA_JSON 4
#define COLUMN_COMPOSITION_SCHEMA_JSON 4
+#define COLUMN_VIRUTAL_SCHEMA_JSON 4
+int read_virtual_table_schema(struct Maat_table_manager* table_mgr, const char* line, struct Maat_table_schema* table, MESA_htable_handle reserved_word_map)
+{
+ int ret=0, tmp_table_id=0;
+ enum MAAT_TABLE_TYPE physical_table_type=TABLE_TYPE_INVALID;
+ enum MAAT_SCAN_TYPE physical_table_scan_type=SCAN_TYPE_INVALID;
+ cJSON* json=NULL, *tmp=NULL;
+ char *json_str;
+
+
+ size_t offset=0, len=0;
+ char* copy_line=NULL;
+ copy_line=_maat_strdup(line);
+ ret=get_column_pos(copy_line, COLUMN_VIRUTAL_SCHEMA_JSON, &offset, &len);
+ if(ret<0)
+ {
+ goto error_out;
+ }
+ if(offset+len<strlen(copy_line))
+ {
+ copy_line[offset+len]='\0';
+ }
+ json_str=copy_line+offset;
+ if(strchr(json_str,'{')||strchr(json_str,'['))//This is a json, mostly.
+ {
+ json=cJSON_Parse(json_str);
+ if(!json)
+ {
+ goto error_out;
+ }
+ if(json->type!=cJSON_Array)
+ {
+ goto error_out;
+ }
+ cJSON_ArrayForEach(tmp, json)
+ {
+ if(tmp->type!=cJSON_String)
+ {
+ goto error_out;
+ }
+ ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &tmp_table_id);
+ if(ret<0)
+ {
+ goto error_out;
+ }
+ physical_table_type=table_mgr->p_table_info[tmp_table_id]->table_type;
+ physical_table_scan_type=Maat_table_get_scan_type(physical_table_type);
+ if(physical_table_scan_type<SCAN_TYPE_IP)
+ {
+ goto error_out;
+ }
+ table->virtual_table.physical_table_id[physical_table_scan_type]=tmp_table_id;
+ }
+ }
+ else //For compatible non-json physical description
+ {
+ ret=map_str2int(table_mgr->map_tablename2id, json_str, &tmp_table_id);
+ if(ret<0)
+ {
+ goto error_out;
+ }
+ physical_table_type=table_mgr->p_table_info[tmp_table_id]->table_type;
+ physical_table_scan_type=Maat_table_get_scan_type(physical_table_type);
+ table->virtual_table.physical_table_id[physical_table_scan_type]=tmp_table_id;
+ }
+ cJSON_Delete(json);
+ free(copy_line);
+ return 0;
+
+error_out:
+ if(json) cJSON_Delete(json);
+ free(copy_line);
+ return -1;
+
+}
int read_plugin_table_schema(const char* line, struct Maat_table_schema* p)
{
@@ -136,7 +232,7 @@ int read_plugin_table_schema(const char* line, struct Maat_table_schema* p)
}
if(offset+len<strlen(copy_line))
{
- copy_line[offset+len+1]='\0';
+ copy_line[offset+len]='\0';
}
plug_info=copy_line+offset;
@@ -173,12 +269,6 @@ int read_plugin_table_schema(const char* line, struct Maat_table_schema* p)
assert(tmp->type==cJSON_Number);
plugin_desc->rule_tag_column=tmp->valueint;
}
- tmp=cJSON_GetObjectItem(json, "estimate_size");
- if(tmp!=NULL)
- {
- assert(tmp->type==cJSON_Number);
- plugin_desc->estimate_size=tmp->valueint;
- }
tmp=cJSON_GetObjectItem(json, "foreign");
if(tmp!=NULL)
{
@@ -220,7 +310,7 @@ int read_ip_plugin_table_schema(const char* line, struct Maat_table_schema* p)
}
if(offset+len<strlen(copy_line))
{
- copy_line[offset+len+1]='\0';
+ copy_line[offset+len]='\0';
}
ip_plugin_info=copy_line+offset;
@@ -271,19 +361,87 @@ int read_ip_plugin_table_schema(const char* line, struct Maat_table_schema* p)
//read_cnt++; Tag is optional, so NOT ++ intentionally.
}
- ip_plugin_schema->estimate_size=4096;
- tmp=cJSON_GetObjectItem(json, "estimate_size");
+ cJSON_Delete(json);
+
+ free(copy_line);
+ if(read_cnt<5)
+ {
+ return -1;
+ }
+ else
+ {
+ return 0;
+ }
+error_out:
+ free(copy_line);
+ return -1;
+
+}
+int read_fqdn_plugin_table_schema(const char* line, struct Maat_table_schema* p)
+{
+ int ret=0, read_cnt=0;
+ size_t offset=0, len=0;
+ cJSON* json=NULL, *tmp=NULL;
+ char* copy_line=NULL, *fqnd_plugin_schema_json=NULL;
+ struct fqdn_plugin_table_schema* fqdn_plugin_schema=&(p->fqdn_plugin);
+
+ copy_line=_maat_strdup(line);
+ ret=get_column_pos(copy_line, COLUMN_FQDN_PLUGIN_SHCEMA_JSON, &offset, &len);
+ if(ret<0)
+ {
+ goto error_out;
+ }
+ if(offset+len<strlen(copy_line))
+ {
+ copy_line[offset+len]='\0';
+ }
+ fqnd_plugin_schema_json=copy_line+offset;
+
+ json=cJSON_Parse(fqnd_plugin_schema_json);
+ if(!json)
+ {
+ goto error_out;
+ }
+ tmp=cJSON_GetObjectItem(json, "row_id");
+ if(tmp!=NULL && tmp->type==cJSON_Number)
+ {
+ fqdn_plugin_schema->row_id_column=tmp->valueint;
+ read_cnt++;
+ }
+
+ tmp=cJSON_GetObjectItem(json, "is_suffix_match");
+ if(tmp!=NULL && tmp->type==cJSON_Number)
+ {
+ fqdn_plugin_schema->is_suffix_flag_column=tmp->valueint;
+ read_cnt++;
+ }
+ tmp=cJSON_GetObjectItem(json, "fqdn");
+ if(tmp!=NULL && tmp->type==cJSON_Number)
+ {
+ fqdn_plugin_schema->fqdn_column=tmp->valueint;
+ read_cnt++;
+ }
+
+ tmp=cJSON_GetObjectItem(json, "valid");
+ if(tmp!=NULL)
+ {
+ assert(tmp->type==cJSON_Number);
+ fqdn_plugin_schema->valid_flag_column=tmp->valueint;
+ read_cnt++;
+ }
+ fqdn_plugin_schema->rule_tag_column=-1;
+ tmp=cJSON_GetObjectItem(json, "tag");
if(tmp!=NULL)
{
assert(tmp->type==cJSON_Number);
- ip_plugin_schema->estimate_size=tmp->valueint;
- //read_cnt++; estimate_size is optional, so NOT ++ intentionally.
+ fqdn_plugin_schema->rule_tag_column=tmp->valueint;
+ //read_cnt++; Tag is optional, so NOT ++ intentionally.
}
cJSON_Delete(json);
free(copy_line);
- if(read_cnt<5)
+ if(read_cnt<4)
{
return -1;
}
@@ -297,7 +455,7 @@ error_out:
}
-int read_composition_table_schema(const char* line, struct Maat_table_schema* p, MESA_htable_handle string2int_map)
+int read_composition_table_schema(struct Maat_table_manager* table_mgr, const char* line, struct Maat_table_schema* p, MESA_htable_handle string2int_map)
{
int ret=0;
size_t offset=0, len=0;
@@ -312,7 +470,7 @@ int read_composition_table_schema(const char* line, struct Maat_table_schema* p,
}
if(offset+len<strlen(copy_line))
{
- copy_line[offset+len+1]='\0';
+ copy_line[offset+len]='\0';
}
composition_info=copy_line+offset;
@@ -325,126 +483,55 @@ int read_composition_table_schema(const char* line, struct Maat_table_schema* p,
tmp=cJSON_GetObjectItem(json, "source");
if(tmp!=NULL && tmp->type==cJSON_String)
{
- strncpy(composition_schema->source_table.real_table_name, tmp->valuestring, sizeof(composition_schema->source_table.real_table_name));
+ ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SOURCE_IP]));
+ if(ret<0)
+ {
+ MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module,
+ "Child table %s of table %s (id=%d) are not defined.",
+ tmp->valuestring,
+ p->table_name[0],
+ p->table_id);
+ goto error_out;
+ }
}
tmp=cJSON_GetObjectItem(json, "destination");
if(tmp!=NULL && tmp->type==cJSON_String)
{
- strncpy(composition_schema->destination_table.real_table_name, tmp->valuestring, sizeof(composition_schema->destination_table.real_table_name));
+ ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_DESTINATION_IP]));
+ if(ret<0)
+ {
+ MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module,
+ "Child table %s of table %s (id=%d) are not defined.",
+ tmp->valuestring,
+ p->table_name[0],
+ p->table_id);
+ goto error_out;
+ }
+
}
tmp=cJSON_GetObjectItem(json, "session");
if(tmp!=NULL && tmp->type==cJSON_String)
{
- strncpy(composition_schema->session_table.real_table_name, tmp->valuestring, sizeof(composition_schema->session_table.real_table_name));
+ ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SESSION]));
+ if(ret<0)
+ {
+ MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module,
+ "Child table %s of table %s (id=%d) are not defined.",
+ tmp->valuestring,
+ p->table_name[0],
+ p->table_id);
+ goto error_out;
+ }
}
cJSON_Delete(json);
-
free(copy_line);
return 0;
+
error_out:
free(copy_line);
return -1;
}
-static int Maat_table_build_map(struct Maat_table_manager* table_mgr, void* logger)
-{
- struct Maat_table_schema** p_table_info=table_mgr->p_table_info;
- size_t n_table=MAX_TABLE_NUM;
-
- MESA_htable_handle map_tablename2id=map_create();
- size_t i=0;
- int j=0, ret=0;
- for(i=0;i<n_table;i++)
- {
- if(p_table_info[i]==NULL)
- {
- continue;
- }
-
- switch(p_table_info[i]->table_type)
- {
- case TABLE_TYPE_VIRTUAL:
- ret=map_str2int(map_tablename2id, p_table_info[i]->virtual_table.real_table_name, &(p_table_info[i]->virtual_table.real_table_id));
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "Undefined real table %s, virtual table %s of table id %d.",
- p_table_info[i]->virtual_table.real_table_name,
- p_table_info[i]->table_name[0],
- p_table_info[i]->table_id);
- goto failed;
- }
- break;
- case TABLE_TYPE_COMPOSITION:
- if(strlen(p_table_info[i]->composition.source_table.real_table_name)>0)
- {
- ret=map_str2int(map_tablename2id, p_table_info[i]->composition.source_table.real_table_name,
- &(p_table_info[i]->composition.source_table.real_table_id));
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "Child table %s of table %s (id=%d) are not defined.",
- p_table_info[i]->composition.source_table.real_table_name,
- p_table_info[i]->table_name[0],
- p_table_info[i]->table_id);
- goto failed;
- }
- }
-
- if(strlen(p_table_info[i]->composition.destination_table.real_table_name)>0)
- {
- ret=map_str2int(map_tablename2id, p_table_info[i]->composition.destination_table.real_table_name,
- &(p_table_info[i]->composition.destination_table.real_table_id));
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "Child table %s of table %s (id=%d) are not defined.",
- p_table_info[i]->composition.destination_table.real_table_name,
- p_table_info[i]->table_name[0],
- p_table_info[i]->table_id);
- goto failed;
- }
- }
- if(strlen(p_table_info[i]->composition.session_table.real_table_name)>0)
- {
- ret=map_str2int(map_tablename2id, p_table_info[i]->composition.session_table.real_table_name,
- &(p_table_info[i]->composition.session_table.real_table_id));
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "Child table %s of table %s (id=%d) are not defined.",
- p_table_info[i]->composition.session_table.real_table_name,
- p_table_info[i]->table_name[0],
- p_table_info[i]->table_id);
- goto failed;
- }
- }
- default:
- break;
- }
-
-
- for(j=0; j<p_table_info[i]->conj_cnt; j++)
- {
- ret=map_register(map_tablename2id, p_table_info[i]->table_name[j], p_table_info[i]->table_id);
- if(ret<0)
- {
- MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
- "Duplicate table %s of table id %d",
- p_table_info[i]->table_name[j],
- p_table_info[i]->table_id);
- continue;
- }
- }
-
- }
- table_mgr->map_tablename2id=map_tablename2id;
- return 0;
-failed:
- map_destroy(map_tablename2id);
- return -1;
-}
-
void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr)
{
size_t i=0;
@@ -469,7 +556,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
char line[MAX_TABLE_LINE_SIZE];
int i=0, ret=0;
char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0};
- MESA_htable_handle string2int_map=NULL;;
+ MESA_htable_handle reserved_word_map=NULL;;
struct Maat_table_schema*p=NULL;
struct Maat_table_schema*conj_table=NULL;
fp=fopen(table_info_path,"r");
@@ -483,37 +570,40 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
table_mgr=ALLOC(struct Maat_table_manager, 1);
struct Maat_table_schema** p_table_info=table_mgr->p_table_info;
size_t n_table=MAX_TABLE_NUM;
+ table_mgr->logger=logger;
+ table_mgr->map_tablename2id=map_create();
- string2int_map=map_create();
- map_register(string2int_map,"expr", TABLE_TYPE_EXPR);
- map_register(string2int_map,"ip", TABLE_TYPE_IP);
- map_register(string2int_map,"ip_plus", TABLE_TYPE_IP_PLUS);
- map_register(string2int_map,"compile", TABLE_TYPE_COMPILE);
- map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN);
- map_register(string2int_map,"ip_plugin", TABLE_TYPE_IP_PLUGIN);
- map_register(string2int_map,"intval", TABLE_TYPE_INTERVAL);
- map_register(string2int_map,"interval", TABLE_TYPE_INTERVAL);
- map_register(string2int_map,"intval_plus", TABLE_TYPE_INTERVAL_PLUS);
- map_register(string2int_map,"interval_plus", TABLE_TYPE_INTERVAL_PLUS);
- map_register(string2int_map,"digest", TABLE_TYPE_DIGEST);
- map_register(string2int_map,"expr_plus", TABLE_TYPE_EXPR_PLUS);
- map_register(string2int_map,"group", TABLE_TYPE_GROUP);
- map_register(string2int_map,"group2group", TABLE_TYPE_GROUP2GROUP);
- map_register(string2int_map,"group2compile", TABLE_TYPE_GROUP2COMPILE);
- map_register(string2int_map,"similar", TABLE_TYPE_SIMILARITY);
- map_register(string2int_map,"virtual", TABLE_TYPE_VIRTUAL);
- map_register(string2int_map,"composition", TABLE_TYPE_COMPOSITION);
- map_register(string2int_map,"quickoff", 0);
- map_register(string2int_map,"quickon", 1);
- map_register(string2int_map,"escape", USER_REGION_ENCODE_ESCAPE);
-// map_register(string2int_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet
+ reserved_word_map=map_create();
+ map_register(reserved_word_map, "expr", TABLE_TYPE_EXPR);
+ map_register(reserved_word_map, "ip", TABLE_TYPE_IP);
+ map_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS);
+ map_register(reserved_word_map, "compile", TABLE_TYPE_COMPILE);
+ map_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN);
+ map_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN);
+ map_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN);
+ map_register(reserved_word_map, "intval", TABLE_TYPE_INTERVAL);
+ map_register(reserved_word_map, "interval", TABLE_TYPE_INTERVAL);
+ map_register(reserved_word_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS);
+ map_register(reserved_word_map, "interval_plus", TABLE_TYPE_INTERVAL_PLUS);
+ map_register(reserved_word_map, "digest", TABLE_TYPE_DIGEST);
+ map_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS);
+ map_register(reserved_word_map, "group", TABLE_TYPE_GROUP);
+ map_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP);
+ map_register(reserved_word_map, "group2compile", TABLE_TYPE_GROUP2COMPILE);
+ map_register(reserved_word_map, "similar", TABLE_TYPE_SIMILARITY);
+ map_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL);
+ map_register(reserved_word_map, "composition", TABLE_TYPE_COMPOSITION);
+ map_register(reserved_word_map, "quickoff", 0);
+ map_register(reserved_word_map, "quickon", 1);
+ map_register(reserved_word_map, "escape", USER_REGION_ENCODE_ESCAPE);
+// map_register(reserved_word_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet
const char** charset_name_list=charset_get_all_name();
for(i=0;i<MAX_CHARSET_NUM;i++)
{
if(strlen(charset_name_list[i])>0)
{
- map_register(string2int_map, charset_name_list[i], i);
+ map_register(reserved_word_map, charset_name_list[i], i);
}
else
{
@@ -521,8 +611,8 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
}
}
- map_register(string2int_map,"yes", 1);
- map_register(string2int_map,"no", 0);
+ map_register(reserved_word_map,"yes", 1);
+ map_register(reserved_word_map,"no", 0);
i=0;
@@ -546,7 +636,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
"Maat read table info %s line %d error: not enough column.",table_info_path,i);
continue;
}
- ret=map_str2int(string2int_map,str_tolower(table_type_str),(int*)&(p->table_type));
+ ret=map_str2int(reserved_word_map,str_tolower(table_type_str),(int*)&(p->table_type));
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
@@ -557,7 +647,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
{
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
- ret=read_expr_table_info(line, p, string2int_map);
+ ret=read_expr_table_info(line, p, reserved_word_map);
if(ret<0)
{
fprintf(stderr,"Maat read table info %s line %d error:illegal column.\n",table_info_path,i);
@@ -570,9 +660,9 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
ret=read_plugin_table_schema(line, p);
if(ret<0)
{
- fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n", table_info_path,i);
+ fprintf(stderr,"Maat read table info %s line %d error:illegal plugin table schema.\n", table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
- "Maat read table info %s line %d error:illegal plugin info.", table_info_path,i);
+ "Maat read table info %s line %d error:illegal plugin table schema.", table_info_path,i);
goto invalid_table;
}
break;
@@ -580,29 +670,39 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
ret=read_ip_plugin_table_schema(line, p);
if(ret<0)
{
- fprintf(stderr,"Maat read table info %s line %d error:illegal ip_plugin info.\n", table_info_path,i);
+ fprintf(stderr,"Maat read table info %s line %d error:illegal ip_plugin table schema.\n", table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
- "Maat read table info %s line %d error:illegal ip_plugin info.", table_info_path,i);
+ "Maat read table info %s line %d error:illegal ip_plugin table schema.", table_info_path,i);
goto invalid_table;
}
- break;
+ break;
+ case TABLE_TYPE_FQDN_PLUGIN:
+ ret=read_fqdn_plugin_table_schema(line, p);
+ if(ret<0)
+ {
+ fprintf(stderr,"Maat read table info %s line %d error:illegal fqdn_plugin table schema.\n", table_info_path,i);
+ MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
+ "Maat read table info %s line %d error:illegal fqdn_plugin table schema.", table_info_path,i);
+ goto invalid_table;
+ }
+ break;
case TABLE_TYPE_COMPOSITION:
- ret=read_composition_table_schema(line, p, string2int_map);
+ ret=read_composition_table_schema(table_mgr, line, p, reserved_word_map);
if(ret<0)
{
- fprintf(stderr,"Maat read table info %s line %d error:illegal composition info.\n", table_info_path,i);
+ fprintf(stderr,"Maat read table info %s line %d error:illegal composition table schema.\n", table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
- "Maat read table info %s line %d error:illegal composition info.", table_info_path,i);
+ "Maat read table info %s line %d error:illegal composition table schema.", table_info_path,i);
goto invalid_table;
}
break;
case TABLE_TYPE_VIRTUAL:
- ret=read_virtual_table_schema(line, p, string2int_map);
+ ret=read_virtual_table_schema(table_mgr, line, p, reserved_word_map);
if(ret<0)
{
- fprintf(stderr,"Maat read table info %s line %d error:illegal virtual info.\n", table_info_path,i);
+ fprintf(stderr,"Maat read table info %s line %d error:illegal virtual table schema.\n", table_info_path,i);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
- "Maat read table info %s line %d error:illegal virtual info.", table_info_path,i);
+ "Maat read table info %s line %d error:illegal virtual table schema.", table_info_path,i);
goto invalid_table;
}
break;
@@ -610,12 +710,13 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
ret=sscanf(not_care,"%[a-z0-9]",tmp_str);
if(ret>0)
{
- ret=map_str2int(string2int_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding));
+ ret=map_str2int(reserved_word_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding));
}
if(ret!=1)
{
p->compile.user_region_encoding=USER_REGION_ENCODE_NONE;
}
+ break;
default:
break;
}
@@ -628,6 +729,15 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
goto invalid_table;
}
+ ret=map_register(table_mgr->map_tablename2id, p->table_name[0], p->table_id);
+ if(ret<0)
+ {
+ MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
+ "Duplicate table %s of table id %d",
+ p->table_name[0],
+ p->table_id);
+ goto invalid_table;
+ }
if(p_table_info[p->table_id]!=NULL)//duplicate table_id,means conjunction table;
{
conj_table=p_table_info[p->table_id];
@@ -647,21 +757,16 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
//use goto to free the conjunctioned table_info
goto invalid_table;
}
-
p_table_info[p->table_id]=p;
table_mgr->table_cnt++;
continue;
+
invalid_table:
table_info_free(p);
p=NULL;
}
fclose(fp);
- ret=Maat_table_build_map(table_mgr, logger);
- if(ret<0)
- {
- return NULL;
- }
- map_destroy(string2int_map);
+ map_destroy(reserved_word_map);
return table_mgr;
}
size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr)
@@ -747,13 +852,13 @@ struct Maat_table_schema * Maat_table_get_by_id_raw(struct Maat_table_manager* t
return table_mgr->p_table_info[table_id];
}
-struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id)
+struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id)
{
-
+ enum MAAT_SCAN_TYPE tab_scan_type;
struct Maat_table_schema **p_table_info=table_mgr->p_table_info;
size_t n_table=MAX_TABLE_NUM;
- struct Maat_table_schema *p_table=NULL, *p_real_table=NULL;
+ struct Maat_table_schema *p_table=NULL, *p_physical_table=NULL;
if((unsigned int) table_id>n_table)
{
return NULL;
@@ -769,23 +874,20 @@ struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager*
}
if(p_table->table_type==TABLE_TYPE_VIRTUAL)
{
- p_real_table=p_table_info[p_table->virtual_table.real_table_id];
+ p_physical_table=p_table_info[p_table->virtual_table.physical_table_id[scan_type]];
*virutal_table_id=table_id;
}
else
{
- p_real_table=p_table;
+ p_physical_table=p_table;
if(virutal_table_id) *virutal_table_id=0;
}
- if(p_real_table->table_type!=expect_type)
+ tab_scan_type=Maat_table_get_scan_type(p_physical_table->table_type);
+ if(tab_scan_type!=scan_type)
{
- if((expect_type==TABLE_TYPE_EXPR && p_real_table->table_type!=TABLE_TYPE_EXPR_PLUS)||
- (expect_type==TABLE_TYPE_IP && p_real_table->table_type!=TABLE_TYPE_IP_PLUS))
- {
- return NULL;
- }
+ return NULL;
}
- return p_real_table;
+ return p_physical_table;
}
int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name)
{
@@ -808,7 +910,7 @@ int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr,
void* u_para)
{
int idx=0;
- struct Maat_table_schema *p_table=Maat_table_get_scan_by_id(table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
+ struct Maat_table_schema *p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL);
struct plugin_table_schema *plugin_desc=&(p_table->plugin);
if(p_table==NULL)
{
@@ -838,7 +940,7 @@ struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_tabl
{
return NULL;
}
- p_table=Maat_table_get_scan_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL);
+ p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL);
if(!p_table)
{
return NULL;
@@ -863,7 +965,7 @@ int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, c
{
return -1;
}
- p_table=Maat_table_get_scan_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL);
+ p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL);
if(!p_table)
{
return -1;
@@ -970,7 +1072,40 @@ int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_sche
new_func, free_func, dup_func, key2index_func, argl, argp);
table_schema->ip_plugin.have_exdata=1;
return 0;
-}
+}
+int Maat_table_fqdn_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
+ Maat_plugin_EX_new_func_t* new_func,
+ Maat_plugin_EX_free_func_t* free_func,
+ Maat_plugin_EX_dup_func_t* dup_func,
+ Maat_plugin_EX_key2index_func_t* key2index_func,
+ long argl, void *argp,
+ void* logger)
+{
+ if(new_func==NULL || free_func==NULL || dup_func==NULL )
+ {
+ assert(0);
+ MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__);
+ return -1;
+ }
+ if(table_schema->table_type!=TABLE_TYPE_FQDN_PLUGIN)
+ {
+ assert(0);
+ MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, target table is not a fqdn_plugin table.", __FUNCTION__);
+ return -1;
+ }
+ if(table_schema->fqdn_plugin.have_exdata)
+ {
+ assert(0);
+ MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data already registed.", __FUNCTION__);
+ return -1;
+ }
+ Maat_table_EX_data_schema_set(&table_schema->fqdn_plugin.ex_schema,
+ new_func, free_func, dup_func, key2index_func, argl, argp);
+ table_schema->fqdn_plugin.have_exdata=1;
+ return 0;
+}
+
+
void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type)
{
table_mgr->active_plugin_table_num=0;
@@ -1067,28 +1202,35 @@ void Maat_table_set_updating_name(struct Maat_table_schema* p_table, const char*
assert(i<=p_table->conj_cnt);
}
-int Maat_table_get_child_id(struct Maat_table_schema* p_table, enum MAAT_TABLE_CHILD_TYPE type)
+int Maat_table_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type)
{
int ret=-1;
+ struct Maat_table_schema* p_table=Maat_table_get_by_id_raw(table_mgr, parent_table_id);
if(p_table->table_type!=TABLE_TYPE_COMPOSITION)
{
return -1;
}
- switch (type)
+ ret=p_table->composition.component_table_id[type];
+ return ret;
+}
+int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p_table)
+{
+ int valid_flag_column=-1;
+ switch(p_table->table_type)
{
- case CHILD_TABLE_TYPE_SOURCE_IP:
- ret=p_table->composition.source_table.real_table_id;
- break;
- case CHILD_TABLE_TYPE_DESTINATION_IP:
- ret=p_table->composition.destination_table.real_table_id;
+ case TABLE_TYPE_PLUGIN:
+ valid_flag_column=p_table->plugin.valid_flag_column;
break;
- case CHILD_TABLE_TYPE_SESSION:
- ret=p_table->composition.session_table.real_table_id;
+ case TABLE_TYPE_IP_PLUGIN:
+ valid_flag_column=p_table->ip_plugin.valid_flag_column;
+ break;
+ case TABLE_TYPE_FQDN_PLUGIN:
+ valid_flag_column=p_table->fqdn_plugin.valid_flag_column;
break;
default:
- assert(0);
+ valid_flag_column=-1;
break;
}
- return ret;
+ return valid_flag_column;
}
diff --git a/src/entry/Maat_table_runtime.cpp b/src/entry/Maat_table_runtime.cpp
index fe0c574..292c3e2 100644
--- a/src/entry/Maat_table_runtime.cpp
+++ b/src/entry/Maat_table_runtime.cpp
@@ -113,6 +113,57 @@ static void destroy_digest_rule(GIE_digest_t*rule)
rule=NULL;
return;
}
+struct FQDN_rule* fqdn_rule_new(unsigned int id, const char* fqdn, size_t fqdn_len, int is_suffix_match)
+{
+ struct FQDN_rule* fqdn_rule=ALLOC(struct FQDN_rule, 1);
+ //Todo: check FQDN format with regex ^([a-zA-Z0-9._-])+$
+ if(fqdn[0]=='.')
+ {
+ fqdn++;
+ fqdn_len--;
+ }
+ if(fqdn[fqdn_len]=='/')
+ {
+ fqdn_len--;
+ }
+ fqdn_rule->FQDN=ALLOC(char, fqdn_len+1);
+ memcpy(fqdn_rule->FQDN, fqdn, fqdn_len);
+ fqdn_rule->len=fqdn_len;
+ fqdn_rule->is_suffix_match=is_suffix_match;
+ fqdn_rule->id=id;
+ return fqdn_rule;
+}
+void fqdn_rule_free(struct FQDN_rule* fqdn_rule)
+{
+ assert(fqdn_rule->user_tag==NULL);
+ free(fqdn_rule->FQDN);
+ fqdn_rule->FQDN=NULL;
+ free(fqdn_rule);
+ return;
+}
+void _notype_fqdn_rule_free(void* p)
+{
+ fqdn_rule_free((struct FQDN_rule*)p);
+ return;
+}
+struct xx_plugin_ex_free_wrapper
+{
+ struct EX_data_rt* ex_data_rt;
+ char* row;
+ size_t key_offset;
+ size_t key_len;
+ void* logger;
+};
+void xx_plugin_ex_data_wrapper_free(void* ex_data)
+{
+ struct xx_plugin_ex_free_wrapper* wrapper=(struct xx_plugin_ex_free_wrapper*)ex_data;
+ EX_data_rt_delete_by_row(wrapper->ex_data_rt, wrapper->row, wrapper->row + wrapper->key_offset, wrapper->key_len, wrapper->logger);
+ free(wrapper->row);
+ wrapper->key_offset=0;
+ wrapper->key_len=0;
+ free(wrapper);
+ return;
+}
static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_schema, int max_thread_num)
{
@@ -127,7 +178,6 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
break;
case TABLE_TYPE_PLUGIN:
table_rt->plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
- table_schema->plugin.estimate_size,
table_schema->plugin.ex_schema.key2index_func,
NULL);
if(table_schema->plugin.have_exdata)
@@ -136,8 +186,7 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
}
break;
case TABLE_TYPE_IP_PLUGIN:
- table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
- table_schema->ip_plugin.estimate_size,
+ table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->ip_plugin.ex_schema.key2index_func,
free);
if(table_schema->ip_plugin.have_exdata)
@@ -146,6 +195,16 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
}
table_rt->ip_plugin.bin=Maat_garbage_bin_new(0);
break;
+ case TABLE_TYPE_FQDN_PLUGIN:
+ table_rt->fqdn_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
+ table_schema->fqdn_plugin.ex_schema.key2index_func,
+ _notype_fqdn_rule_free);
+ if(table_schema->fqdn_plugin.have_exdata)
+ {
+ EX_data_rt_set_schema(table_rt->fqdn_plugin.ex_data_rt, &table_schema->fqdn_plugin.ex_schema);
+ }
+ table_rt->fqdn_plugin.bin=Maat_garbage_bin_new(0);
+ break;
default:
break;
}
@@ -196,6 +255,12 @@ static void table_runtime_free(struct Maat_table_runtime* p)
EX_data_rt_free(p->ip_plugin.ex_data_rt);
assert(p->ip_plugin.new_ip_matcher==NULL);
break;
+ case TABLE_TYPE_FQDN_PLUGIN:
+ FQDN_engine_free(p->fqdn_plugin.fqdn_engine);
+ Maat_garbage_bin_free(p->fqdn_plugin.bin);
+ EX_data_rt_free(p->fqdn_plugin.ex_data_rt);
+ assert(p->fqdn_plugin.new_fqdn_engine==NULL);
+ break;
case TABLE_TYPE_PLUGIN:
EX_data_rt_free(p->plugin.ex_data_rt);
break;
@@ -358,6 +423,156 @@ void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr
MESA_lqueue_join_tail(table_rt->similar.update_q,&digest_rule, sizeof(void*));
return;
}
+
+void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger)
+{
+ struct fqdn_plugin_table_schema* fqdn_plugin_schema=&(table_schema->fqdn_plugin);
+ struct fqdn_plugin_runtime* fqdn_plugin_rt=&(table_rt->fqdn_plugin);
+ size_t is_valid_offset=0, valid_len=0;
+ size_t is_suffix_flag_offset=0, is_suffix_flag_len=0;
+ size_t row_id_offset=0, row_id_len=0;
+ size_t fqdn_offset=0, fqdn_len=0;
+ struct FQDN_rule* fqdn_rule=NULL;
+ int ret=0;
+ struct xx_plugin_ex_free_wrapper* wrapper_for_free=NULL;
+ if(fqdn_plugin_schema->have_exdata)
+ {
+ ret=Maat_helper_read_column(row, fqdn_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
+ if(ret<0)
+ {
+ MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
+ "fqdn_plugin EX data process error: cannot find is_valid column %d of %s",
+ fqdn_plugin_schema->valid_flag_column, row);
+ return;
+ }
+ ret=Maat_helper_read_column(row, fqdn_plugin_schema->row_id_column, &row_id_offset, &row_id_len);
+ if(ret<0)
+ {
+ MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
+ "fqdn_plugin EX data process error: cannot find row id column %d of %s",
+ fqdn_plugin_schema->row_id_column, row);
+ return;
+ }
+ ret=Maat_helper_read_column(row, fqdn_plugin_schema->is_suffix_flag_column, &is_suffix_flag_offset, &is_suffix_flag_len);
+ if(ret<0)
+ {
+ MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
+ "fqdn_plugin EX data process error: cannot find is_suffix_match column %d of %s",
+ fqdn_plugin_schema->is_suffix_flag_column, row);
+ return;
+ }
+ ret=Maat_helper_read_column(row, fqdn_plugin_schema->fqdn_column, &fqdn_offset, &fqdn_len);
+ if(ret<0)
+ {
+ MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
+ "fqdn_plugin EX data process error: cannot find fqdn column %d of %s",
+ fqdn_plugin_schema->fqdn_column, row);
+ return;
+ }
+ fqdn_rule=fqdn_rule_new((unsigned int)atoi(row+row_id_offset), row+fqdn_offset, fqdn_len, atoi(row+is_suffix_flag_offset));
+
+ if(atoi(row+is_valid_offset)==1)//add
+ {
+
+ EX_data_rt_row2EX_data(fqdn_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, fqdn_rule, logger);
+ }
+ else
+ {
+ wrapper_for_free=ALLOC(struct xx_plugin_ex_free_wrapper, 1);
+ wrapper_for_free->row=_maat_strdup(row);
+ wrapper_for_free->ex_data_rt=fqdn_plugin_rt->ex_data_rt;
+ wrapper_for_free->key_len=row_id_len;
+ wrapper_for_free->key_offset=row_id_offset;
+ wrapper_for_free->logger=logger;
+ Maat_garbage_bagging(fqdn_plugin_rt->bin, wrapper_for_free, xx_plugin_ex_data_wrapper_free);
+ }
+ }
+ else
+ {
+ EX_data_rt_cache_row_put(fqdn_plugin_rt->ex_data_rt, row);
+ }
+ fqdn_plugin_rt->changed_flag=1;
+ return;
+}
+
+int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt)
+{
+ struct FQDN_engine* new_fqdn_engine=NULL;
+ struct fqdn_plugin_runtime* fqdn_rt=&table_rt->fqdn_plugin;
+ assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN);
+ struct EX_data_container **exc_array=NULL;
+ Maat_garbage_collect_routine(fqdn_rt->bin);
+ struct FQDN_rule* rules=NULL;
+ size_t rule_cnt=0, i=0;
+ if(!fqdn_rt->changed_flag)
+ {
+ assert(0==Maat_garbage_bin_get_size(fqdn_rt->bin));
+ return 0;
+ }
+
+ rule_cnt=EX_data_rt_list_all_ex_container(fqdn_rt->ex_data_rt, &exc_array);
+ rules=ALLOC(struct FQDN_rule, rule_cnt);
+ for(i=0; i<rule_cnt; i++)
+ {
+ rules[i]=*((struct FQDN_rule *)(EX_data_container_get_user_data(exc_array[i])));
+ assert(rules[i].user_tag==exc_array[i]||rules[i].user_tag==NULL);
+ rules[i].user_tag=exc_array[i];
+ }
+ if(rule_cnt>0)
+ {
+ new_fqdn_engine=FQDN_engine_new(rules, rule_cnt);
+ }
+ fqdn_rt->new_fqdn_engine=new_fqdn_engine;
+ free(rules);
+ free(exc_array);
+ return 1;
+}
+struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_runtime* table_rt)
+{
+ struct FQDN_engine* old_one=table_rt->fqdn_plugin.fqdn_engine;
+ table_rt->fqdn_plugin.fqdn_engine=table_rt->fqdn_plugin.new_fqdn_engine;
+ assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN);
+ table_rt->fqdn_plugin.new_fqdn_engine=NULL;
+ table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->fqdn_plugin.ex_data_rt);
+ return old_one;
+}
+
+int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger)
+{
+ size_t i=0;
+ const char* row=NULL;
+ struct fqdn_plugin_runtime* fqdn_plugin_rt=&(table_rt->fqdn_plugin);
+ EX_data_rt_set_schema(fqdn_plugin_rt->ex_data_rt, &table_schema->fqdn_plugin.ex_schema);
+ for(i=0; i<EX_data_rt_get_cached_row_num(fqdn_plugin_rt->ex_data_rt); i++)
+ {
+ row=EX_data_rt_cached_row_get(fqdn_plugin_rt->ex_data_rt, i);
+ Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, row, logger);
+ }
+ EX_data_rt_clear_row_cache(fqdn_plugin_rt->ex_data_rt);
+ Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(table_rt);
+ Maat_table_runtime_apply_new_fqdn_engine(table_rt);//returned NULL.
+ return 0;
+}
+int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size)
+{
+ struct FQDN_match results[size];
+ int n_result=0, i=0;
+ if(table_rt->table_type!=TABLE_TYPE_FQDN_PLUGIN)
+ {
+ return -1;
+ }
+ if(!table_rt->fqdn_plugin.fqdn_engine)
+ {
+ return 0;
+ }
+ n_result=FQDN_engine_search(table_rt->fqdn_plugin.fqdn_engine, query_fqdn, strlen(query_fqdn), results, size);
+ for(i=0; i<n_result; i++)
+ {
+ ex_data_array[i]=EX_data_rt_get_EX_data_by_container(table_rt->fqdn_plugin.ex_data_rt, (struct EX_data_container *)results[i].user_tag);
+ }
+ return n_result;
+}
+
int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt)
{
long i=0,data_size=0;
@@ -459,41 +674,28 @@ int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime*
ip_plugin->changed_flag=0;
return 1;
}
+
+
+
struct ip_matcher* Maat_table_runtime_apply_new_ip_matcher(struct Maat_table_runtime* table_rt)
{
struct ip_matcher* old_one=table_rt->ip_plugin.ip_matcher;
table_rt->ip_plugin.ip_matcher=table_rt->ip_plugin.new_ip_matcher;
assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN);
table_rt->ip_plugin.new_ip_matcher=NULL;
+ table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->ip_plugin.ex_data_rt);
return old_one;
}
-struct ip_plugin_ex_free_wrapper
-{
- struct EX_data_rt* ex_data_rt;
- char* row;
- size_t key_offset;
- size_t key_len;
- void* logger;
-};
-void ip_plugin_ex_data_wrapper_free(void* ex_data)
-{
- struct ip_plugin_ex_free_wrapper* wrapper=(struct ip_plugin_ex_free_wrapper*)ex_data;
- EX_data_rt_delete_by_row(wrapper->ex_data_rt, wrapper->row, wrapper->row + wrapper->key_offset, wrapper->key_len, wrapper->logger);
- free(wrapper->row);
- wrapper->key_offset=0;
- wrapper->key_len=0;
- free(wrapper);
- return;
-}
+
void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger)
{
struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin);
struct ip_plugin_runtime* ip_plugin_rt=&(table_rt->ip_plugin);
size_t is_valid_offset=0, valid_len=0;
- size_t key_offset=0, key_len=0;
+ size_t row_id_offset=0, row_id_len=0;
struct ip_rule* ip_rule=NULL;
int ret=0;
- struct ip_plugin_ex_free_wrapper* wrapper_for_free=NULL;
+ struct xx_plugin_ex_free_wrapper* wrapper_for_free=NULL;
if(ip_plugin_schema->have_exdata)
{
ret=Maat_helper_read_column(row, ip_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
@@ -504,7 +706,7 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s
ip_plugin_schema->row_id_column, row);
return;
}
- ret=Maat_helper_read_column(row, ip_plugin_schema->row_id_column, &key_offset, &key_len);
+ ret=Maat_helper_read_column(row, ip_plugin_schema->row_id_column, &row_id_offset, &row_id_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
@@ -523,17 +725,18 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s
if(atoi(row+is_valid_offset)==1)//add
{
- EX_data_rt_row2EX_data(ip_plugin_rt->ex_data_rt, row, row+key_offset, key_len, ip_rule, logger);
+ EX_data_rt_row2EX_data(ip_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, ip_rule, logger);
}
else
{
- wrapper_for_free=ALLOC(struct ip_plugin_ex_free_wrapper, 1);
+ wrapper_for_free=ALLOC(struct xx_plugin_ex_free_wrapper, 1);
wrapper_for_free->row=_maat_strdup(row);
wrapper_for_free->ex_data_rt=ip_plugin_rt->ex_data_rt;
- wrapper_for_free->key_len=key_len;
- wrapper_for_free->key_offset=key_offset;
+ wrapper_for_free->key_len=row_id_len;
+ wrapper_for_free->key_offset=row_id_offset;
wrapper_for_free->logger=logger;
- Maat_garbage_bagging(ip_plugin_rt->bin, wrapper_for_free, ip_plugin_ex_data_wrapper_free);
+ Maat_garbage_bagging(ip_plugin_rt->bin, wrapper_for_free, xx_plugin_ex_data_wrapper_free);
+ free(ip_rule);
}
}
else
diff --git a/src/inc_internal/FQDN_engine.h b/src/inc_internal/FQDN_engine.h
new file mode 100644
index 0000000..d2213fc
--- /dev/null
+++ b/src/inc_internal/FQDN_engine.h
@@ -0,0 +1,69 @@
+/*
+ *
+ * Copyright (c) 2020
+ * String Algorithms Research Group
+ * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS)
+ * National Engineering Laboratory for Information Security Technologies (NELIST)
+ * All rights reserved
+ *
+ * Written by: LIU YANBING ([email protected])
+ * Last modification: 2020-09-01
+ *
+ * This code is the exclusive and proprietary property of IIE-CAS and NELIST.
+ * Usage for direct or indirect commercial advantage is not allowed without
+ * written permission from the authors.
+ *
+ */
+
+#ifndef H_FQDN_ENGINE_H
+#define H_FQDN_ENGINE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+ #include <stddef.h>
+
+ struct FQDN_rule
+ {
+ unsigned int id;
+ int is_suffix_match; /* is_suffix_match==0: exact match; is_suffix_match==1: longest suffix matching. */
+ size_t len;
+ char * FQDN; /* Non-ASCII character is allowed. */
+ void * user_tag; /* A transparent user tag for convenient accessing, the caller is responsible for its memory management. */
+ };
+
+ struct FQDN_engine;
+
+ struct FQDN_engine * FQDN_engine_new(const struct FQDN_rule * rules, size_t n_rule);
+
+ struct FQDN_match
+ {
+ unsigned int id;
+ unsigned int offset; /* offset==0 for exact matching; offset>0 for longest suffix matching. */
+ void * user_tag;
+ };
+
+ /*
+ *Function:
+ * Search FQDN in the rule base
+ *Paramters:
+ * instance[in]: Instance of FQDN engine
+ * FQDN[in]: FQDN for search
+ * FQDN_len[in]: Length of FQDN
+ * results[out]: An array to store matched FQDNs
+ * n_result[in]: Number of element in the result array
+ * Return:
+ * 0: No matched FQDN;
+ * >0: Number of matched FQNDs which were stored in results;
+ * <0: Error.
+ */
+ int FQDN_engine_search(struct FQDN_engine * instance, const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result);
+
+ void FQDN_engine_free(struct FQDN_engine * instance);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/inc_internal/Maat_ex_data.h b/src/inc_internal/Maat_ex_data.h
index baa3d15..17230ab 100644
--- a/src/inc_internal/Maat_ex_data.h
+++ b/src/inc_internal/Maat_ex_data.h
@@ -3,7 +3,7 @@
struct EX_data_rt;
-struct EX_data_rt* EX_data_rt_new(int table_id, long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data));
+struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data));
void EX_data_rt_free(struct EX_data_rt* p);
void EX_data_rt_set_schema(struct EX_data_rt* p, const struct EX_data_schema* schema);
void EX_data_rt_cache_row_put(struct EX_data_rt* p, const char* row);
@@ -23,4 +23,5 @@ MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const
MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, struct EX_data_container* container);
size_t EX_data_rt_list_all_ex_container(struct EX_data_rt* ex_rt, struct EX_data_container*** ex_container_array);
void* EX_data_container_get_user_data(struct EX_data_container* ex_container);
+size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt);
diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h
index 39016f3..dba92e2 100644
--- a/src/inc_internal/Maat_rule_internal.h
+++ b/src/inc_internal/Maat_rule_internal.h
@@ -16,6 +16,7 @@
#include "stream_fuzzy_hash.h"
#include "gram_index_engine.h"
#include "alignment_int64.h"
+#include "FQDN_engine.h"
#include <pthread.h>
#include <iconv.h>
#include <openssl/md5.h>
@@ -78,6 +79,15 @@ struct db_digest_rule
short confidence_degree;
int is_valid;
};
+struct db_fqdn_rule
+{
+ int region_id;
+ int group_id;
+ int is_suffix_match;
+ char* fqdn;
+ int is_valid;
+};
+
struct Maat_rule_head
{
int config_id;
@@ -204,7 +214,7 @@ struct Maat_scanner
mcore_long_t ref_cnt;
rule_scanner_t region;
size_t gie_update_q_size;
- size_t ip_plugin_update_q_size;
+ size_t xx_plugin_update_q_size;
size_t to_update_group_cnt;
size_t to_update_compile_cnt;
diff --git a/src/inc_internal/Maat_table.h b/src/inc_internal/Maat_table.h
index f607827..c34a1de 100644
--- a/src/inc_internal/Maat_table.h
+++ b/src/inc_internal/Maat_table.h
@@ -17,6 +17,28 @@ enum USER_REGION_ENCODE
USER_REGION_ENCODE_ESCAPE,
USER_REGION_ENCODE_BASE64
};
+enum MAAT_SCAN_TYPE
+{
+ SCAN_TYPE_INVALID=-1,
+ SCAN_TYPE_NONE=0,
+ SCAN_TYPE_PLUGIN,
+ SCAN_TYPE_IP_PLUGIN,
+ SCAN_TYPE_FQDN_PLUGIN,
+ SCAN_TYPE_IP,
+ SCAN_TYPE_INTERVAL,
+ SCAN_TYPE_STRING,
+ __SCAN_TYPE_MAX
+};
+
+enum MAAT_TABLE_COMPONENT_TYPE
+{
+ COMPONENT_TABLE_TYPE_NONE=-1,
+ COMPONENT_TABLE_TYPE_SOURCE_IP=0,
+ COMPONENT_TABLE_TYPE_DESTINATION_IP,
+ COMPONENT_TABLE_TYPE_SESSION,
+ __COMPONENT_TABLE_TYPE_MAX
+};
+
enum MAAT_TABLE_TYPE
{
@@ -28,6 +50,8 @@ enum MAAT_TABLE_TYPE
TABLE_TYPE_DIGEST,
TABLE_TYPE_EXPR_PLUS,
TABLE_TYPE_SIMILARITY,
+ TABLE_TYPE_INTERVAL_PLUS,
+ //Above are physical table for scan
TABLE_TYPE_VIRTUAL,
TABLE_TYPE_COMPOSITION,
TABLE_TYPE_GROUP2GROUP,
@@ -36,7 +60,7 @@ enum MAAT_TABLE_TYPE
TABLE_TYPE_COMPILE,
TABLE_TYPE_PLUGIN,
TABLE_TYPE_IP_PLUGIN,
- TABLE_TYPE_INTERVAL_PLUS
+ TABLE_TYPE_FQDN_PLUGIN
};
struct compile_ex_data_idx
@@ -68,12 +92,11 @@ struct expr_table_schema
};
struct virtual_table_schema
{
- int real_table_id;
- char real_table_name[MAX_TABLE_NAME_LEN];
+ int physical_table_id[__SCAN_TYPE_MAX];
};
struct composition_table_schema
{
- struct virtual_table_schema source_table, destination_table, session_table;
+ int component_table_id[__COMPONENT_TABLE_TYPE_MAX];
};
struct plugin_table_callback_schema
{
@@ -100,7 +123,6 @@ struct plugin_table_schema
int foreign_columns[MAX_FOREIGN_CLMN_NUM];
int cb_plug_cnt;
int have_exdata;
- long long estimate_size;
struct plugin_table_callback_schema cb_plug[MAX_PLUGIN_PER_TABLE];
struct EX_data_schema ex_schema;
};
@@ -112,7 +134,17 @@ struct ip_plugin_table_schema
int end_ip_column;
int valid_flag_column;
int rule_tag_column;
- long long estimate_size;
+ int have_exdata;
+ struct EX_data_schema ex_schema;
+};
+struct fqdn_plugin_table_schema
+{
+
+ int row_id_column;
+ int is_suffix_flag_column;
+ int fqdn_column;
+ int valid_flag_column;
+ int rule_tag_column;
int have_exdata;
struct EX_data_schema ex_schema;
};
@@ -129,6 +161,7 @@ struct Maat_table_schema
struct expr_table_schema expr;
struct plugin_table_schema plugin;
struct ip_plugin_table_schema ip_plugin;
+ struct fqdn_plugin_table_schema fqdn_plugin;
struct virtual_table_schema virtual_table;
struct composition_table_schema composition;
void* others;//group, ip, interval and digest don't have any special schema.
@@ -144,8 +177,9 @@ void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr);
size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr);
size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr);
-struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id);
+struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id);
struct Maat_table_schema * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id);
+enum MAAT_SCAN_TYPE Maat_table_get_scan_type(enum MAAT_TABLE_TYPE table_type);
int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr,
@@ -181,6 +215,13 @@ int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_sche
Maat_plugin_EX_key2index_func_t* key2index_func,
long argl, void *argp,
void* logger);
+int Maat_table_fqdn_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema,
+ Maat_plugin_EX_new_func_t* new_func,
+ Maat_plugin_EX_free_func_t* free_func,
+ Maat_plugin_EX_dup_func_t* dup_func,
+ Maat_plugin_EX_key2index_func_t* key2index_func,
+ long argl, void *argp,
+ void* logger);
void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type);
void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr);
@@ -188,12 +229,7 @@ void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mg
int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr);
struct Maat_table_schema* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
void Maat_table_set_updating_name(struct Maat_table_schema* p_table, const char* table_name);
-enum MAAT_TABLE_CHILD_TYPE
-{
- CHILD_TABLE_TYPE_NONE=-1,
- CHILD_TABLE_TYPE_SOURCE_IP=0,
- CHILD_TABLE_TYPE_DESTINATION_IP,
- CHILD_TABLE_TYPE_SESSION
-};
-int Maat_table_get_child_id(struct Maat_table_schema* p_table, enum MAAT_TABLE_CHILD_TYPE type);
+
+int Maat_table_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type);
+int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p_table);
diff --git a/src/inc_internal/Maat_table_runtime.h b/src/inc_internal/Maat_table_runtime.h
index 15fc7a6..0ac2846 100644
--- a/src/inc_internal/Maat_table_runtime.h
+++ b/src/inc_internal/Maat_table_runtime.h
@@ -4,6 +4,7 @@
#include "IPMatcher.h"
#include "gram_index_engine.h"
+#include "FQDN_engine.h"
#include "alignment_int64.h"
#include "dynamic_array.h"
#include <MESA/MESA_htable.h>
@@ -14,7 +15,14 @@ struct similar_runtime
GIE_handle_t* gie_handle;
MESA_lqueue_head update_q;
};
-
+struct fqdn_plugin_runtime
+{
+ struct FQDN_engine* fqdn_engine;
+ struct FQDN_engine* new_fqdn_engine;
+ struct EX_data_rt* ex_data_rt; //for fqdn_plugin ONLY
+ struct Maat_garbage_bin* bin;
+ int changed_flag;
+};
struct plugin_runtime
{
struct EX_data_rt* ex_data_rt;
@@ -51,6 +59,7 @@ struct Maat_table_runtime
union
{
struct similar_runtime similar; //for digest and similarity
+ struct fqdn_plugin_runtime fqdn_plugin;//for fqdn_plugin and fqdn_plugin
struct plugin_runtime plugin;
struct ip_plugin_runtime ip_plugin;
struct expr_runtime expr;
@@ -86,4 +95,12 @@ int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_
int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime* table_rt);
struct ip_matcher* Maat_table_runtime_apply_new_ip_matcher(struct Maat_table_runtime* table_rt);
+int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt);
+
+void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);
+int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt);
+struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_runtime* table_rt);
+int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger);
+int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size);
+
diff --git a/test/maat_json.json b/test/maat_json.json
index f7d8d59..8f835c3 100644
--- a/test/maat_json.json
+++ b/test/maat_json.json
@@ -60,7 +60,7 @@
"regions": [
{
"table_name": "GeoLocation",
- "table_type": "string",
+ "table_type": "expr",
"table_content": {
"keywords": "Greece.Sparta",
"expr_type": "none",
@@ -69,6 +69,35 @@
}
}
]
+ },
+ {
+ "group_name": "FQDN_OBJ1",
+ "regions": [
+ {
+ "table_name": "KEYWORDS_TABLE",
+ "table_type": "expr",
+ "table_content": {
+ "keywords": "sports.example.com",
+ "expr_type": "none",
+ "match_method": "exact",
+ "format": "uncase plain"
+ }
+ }
+ ]
+ },
+ {
+ "group_name": "FQDN_CAT1",
+ "regions": [
+ {
+ "table_name": "INTERGER_PLUS",
+ "table_type": "intval_plus",
+ "table_content": {
+ "district": "fqdn_cat_id",
+ "low_boundary": 1724,
+ "up_boundary": 1724
+ }
+ }
+ ]
}
],
"rules": [
@@ -1220,7 +1249,7 @@
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
- "user_region": "VirtualWithPhysical",
+ "user_region": "VirtualWithOnePhysical",
"is_valid": "yes",
"groups": [
{
@@ -1850,51 +1879,85 @@
]
}
]
- }
+ },
+ {
+ "compile_id": 180,
+ "service": 0,
+ "action": 0,
+ "do_blacklist": 0,
+ "do_log": 0,
+ "effective_rage": 0,
+ "user_region": "Hierarchy_VirtualWithTwoPhysical",
+ "is_valid": "yes",
+ "groups": [
+ {
+ "group_name":"FQDN_OBJ1",
+ "virtual_table":"VIRTUAL_SSL_SNI",
+ "not_flag" : 0,
+ "clause_index":0
+ },
+ {
+ "group_name":"FQDN_CAT1",
+ "virtual_table":"VIRTUAL_SSL_SNI",
+ "not_flag" : 0,
+ "clause_index":0
+ }
+ ]
+ }
],
"plugin_table": [
{
"table_name": "QD_ENTRY_INFO",
"table_content": [
- "1\t192.168.0.1\t101\t1",
- "2\t192.168.0.2\t101\t1",
- "3\t192.168.1.1\t102\t1"
+ "1\t192.168.0.1\t101\t1",
+ "2\t192.168.0.2\t101\t1",
+ "3\t192.168.1.1\t102\t1"
]
},
{
"table_name": "TEST_PLUGIN_TABLE",
"table_content": [
- "1\t3388\t99\t1",
- "2\t3355\t66\t1",
- "3\tcccc\t11\t1"
+ "1\t3388\t99\t1",
+ "2\t3355\t66\t1",
+ "3\tcccc\t11\t1"
]
},
{
"table_name": "TEST_EFFECTIVE_RANGE_TABLE",
"table_content": [
- "1\tSUCCESS\t99\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京/朝阳/华严北里\"]},{\"tag\":\"isp\",\"value\":[\"电信\",\"移动\"]}]]}\t1111",
- "2\tSUCCESS\t66\t1\t0\t222",
- "3\tFAILED\t11\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京/朝阳/华严北里\",\"上海/浦东/陆家嘴\"]},{\"tag\":\"isp\",\"value\":[\"电信\",\"联通\"]}],[{\"tag\":\"location\",\"value\":[\"北京\"]},{\"tag\":\"isp\",\"value\":[\"联通\"]}]]}\t333",
- "4\tSUCCESS\t66\t1\t{}\t444",
- "5\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京\"]}]]}\t444",
- "6\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"weather\",\"value\":[\"hot\"]}]]}\t444"
+ "1\tSUCCESS\t99\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京/朝阳/华严北里\"]},{\"tag\":\"isp\",\"value\":[\"电信\",\"移动\"]}]]}\t1111",
+ "2\tSUCCESS\t66\t1\t0\t222",
+ "3\tFAILED\t11\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京/朝阳/华严北里\",\"上海/浦东/陆家嘴\"]},{\"tag\":\"isp\",\"value\":[\"电信\",\"联通\"]}],[{\"tag\":\"location\",\"value\":[\"北京\"]},{\"tag\":\"isp\",\"value\":[\"联通\"]}]]}\t333",
+ "4\tSUCCESS\t66\t1\t{}\t444",
+ "5\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"北京\"]}]]}\t444",
+ "6\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"weather\",\"value\":[\"hot\"]}]]}\t444"
]
},
{
"table_name": "IR_INTERCEPT_IP",
"table_content": [
- "1000000130\t1000000130\t4\t192.168.10.99\t255.255.255.255\t0\t65535\t0.0.0.0\t255.255.255.255\t0\t65535\t0\t1\t1\t96\t1\tuser_region\t{}\t2019/1/24/18:0:34",
- "161\t161\t4\t0.0.0.0\t255.255.255.255\t0\t65535\t61.135.169.121\t255.255.255.255\t0\t65535\t0\t0\t1\t96\t832\t0\t0\t2019/1/24/18:48:42"
+ "1000000130\t1000000130\t4\t192.168.10.99\t255.255.255.255\t0\t65535\t0.0.0.0\t255.255.255.255\t0\t65535\t0\t1\t1\t96\t1\tuser_region\t{}\t2019/1/24/18:0:34",
+ "161\t161\t4\t0.0.0.0\t255.255.255.255\t0\t65535\t61.135.169.121\t255.255.255.255\t0\t65535\t0\t0\t1\t96\t832\t0\t0\t2019/1/24/18:48:42"
]
},
{
"table_name": "TEST_IP_PLUGIN_WITH_EXDATA",
"table_content": [
- "101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1",
+ "101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1",
"102\t4\t192.168.30.90\t192.168.30.128\tBigger-range-should-in-the-back\t1",
- "103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1",
+ "103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1",
"104\t6\t2001:db8:1234::1\t2001:db8:1234::5210\tSomething-like-json\t1"
]
+ },
+ {
+ "table_name": "TEST_FQDN_PLUGIN_WITH_EXDATA",
+ "table_content": [
+ "201\t0\twww.example1.com\tcatid=1\t1",
+ "202\t1\t.example1.com\tcatid=1\t1",
+ "203\t0\tnews.example1.com\tcatid=2\t1",
+ "204\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1",
+ "205\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1"
+ ]
}
]
}
diff --git a/test/table_info.conf b/test/table_info.conf
index 64387a3..adfaf31 100644
--- a/test/table_info.conf
+++ b/test/table_info.conf
@@ -55,4 +55,6 @@
32 DESTINATION_IP_ASN virtual AS_NUMBER --
33 GeoLocation expr UTF8 UTF8 yes 0
34 SOURCE_IP_GEO virtual GeoLocation --
-35 INTERGER_PLUS intval_plus -- \ No newline at end of file
+35 INTERGER_PLUS intval_plus --
+36 TEST_FQDN_PLUGIN_WITH_EXDATA fqdn_plugin {"row_id":1,"is_suffix_match":2,"fqdn":3,"valid":5} --
+37 VIRTUAL_SSL_SNI virtual ["KEYWORDS_TABLE","INTERGER_PLUS"] -- \ No newline at end of file
diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp
index 6d9e01a..2e6876e 100644
--- a/test/test_maatframe.cpp
+++ b/test/test_maatframe.cpp
@@ -310,6 +310,86 @@ TEST(IP_Plugin_Table, EX_DATA)
}
+#define FQDN_PLUGIN_EX_DATA
+struct fqdn_plugin_ud
+{
+ int rule_id;
+ int catid;
+ int ref_cnt;
+};
+void fqdn_plugin_EX_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
+{
+ int *counter=(int *)argp, ret=0;
+ size_t column_offset=0, column_len=0;
+ struct fqdn_plugin_ud* ud=(struct fqdn_plugin_ud*)calloc(sizeof(struct fqdn_plugin_ud), 1);
+ ret=Maat_helper_read_column(table_line, 1, &column_offset, &column_len);
+ EXPECT_EQ(ret, 0);
+ ud->rule_id=atoi(table_line+column_offset);
+ ret=Maat_helper_read_column(table_line, 4, &column_offset, &column_len);
+ EXPECT_EQ(ret, 0);
+ sscanf(table_line+column_offset, "catid=%d",&ud->catid);
+ ud->ref_cnt=1;
+ *ad=ud;
+ (*counter)++;
+ return;
+}
+void fqdn_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
+{
+ struct fqdn_plugin_ud* u=(struct fqdn_plugin_ud*)(*ad);
+ u->ref_cnt--;
+ if(u->ref_cnt>0) return;
+ free(u);
+ *ad=NULL;
+}
+void fqdn_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp)
+{
+ struct fqdn_plugin_ud* u=(struct fqdn_plugin_ud*)(*from);
+ u->ref_cnt++;
+ *to=u;
+}
+TEST(FQDN_Plugin_Table, EX_DATA)
+{
+
+ int fqdn_plugin_ex_data_counter=0, i=0;
+ const char* table_name="TEST_FQDN_PLUGIN_WITH_EXDATA";
+ int table_id=0, ret=0;
+ table_id=Maat_table_register(g_feather, table_name);
+ ASSERT_GT(table_id, 0);
+ ret=Maat_fqdn_plugin_EX_register(g_feather, table_id,
+ fqdn_plugin_EX_new_cb,
+ fqdn_plugin_EX_free_cb,
+ fqdn_plugin_EX_dup_cb,
+ 0, &fqdn_plugin_ex_data_counter);
+ ASSERT_TRUE(ret>=0);
+ EXPECT_EQ(fqdn_plugin_ex_data_counter, 5);
+
+ struct fqdn_plugin_ud* result[4];
+
+ ret=Maat_fqdn_plugin_get_EX_data(g_feather, table_id, "www.example1.com", (void**)result, 4);
+ ASSERT_EQ(ret, 2);
+ EXPECT_EQ(result[0]->rule_id, 201);
+ EXPECT_EQ(result[1]->rule_id, 202);
+
+ for(i=0; i<ret; i++)
+ {
+ fqdn_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL);
+ }
+
+ ret=Maat_fqdn_plugin_get_EX_data(g_feather, table_id, "www.example3.com", (void**)result, 4);
+ EXPECT_EQ(ret, 0);
+
+ ret=Maat_fqdn_plugin_get_EX_data(g_feather, table_id, "r3---sn-i3belne6.example2.com", (void**)result, 4);
+ ASSERT_EQ(ret, 2);
+ EXPECT_EQ(result[0]->rule_id, 205);
+ EXPECT_EQ(result[1]->rule_id, 204);
+ for(i=0; i<ret; i++)
+ {
+ fqdn_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL);
+ }
+
+}
+
+
TEST(StringScan, Full)
{
int ret=0;
@@ -1755,9 +1835,9 @@ TEST(ScanResult, LongerServiceDefine)
free(buff);
return;
}
-TEST(Hierarchy, VirtualWithPhysical)
+TEST(Hierarchy, VirtualOfOnePhysical)
{
-#define Hierarchy_VirtualWithPhysical
+#define Hierarchy_VirtualOfOnePhysical
int ret=0, table_id=0;
const char* http_content="Batman\\:Take me Home.Superman/:Fine,stay with me.";
const char* http_url="https://blog.csdn.net/littlefang/article/details/8213058";
@@ -1796,6 +1876,42 @@ TEST(Hierarchy, VirtualWithPhysical)
return;
}
+TEST(Hierarchy, VirtualOfTwoPhysical)
+{
+#define Hierarchy_VirtualWithTwoPhysical
+ int ret=0, table_id=0;
+ const char* sni="sports.example.com";
+ int cat_id=1724;
+
+
+ struct Maat_rule_t result[4];
+ memset(result, 0, sizeof(result));
+
+ scan_status_t mid=NULL;
+
+ table_id=Maat_table_register(g_feather, "VIRTUAL_SSL_SNI");
+ ASSERT_GT(table_id, 0);
+
+ ret=Maat_full_scan_string(g_feather, table_id, CHARSET_UTF8, sni, strlen(sni),
+ result, NULL, 4, &mid, 0);
+
+ EXPECT_EQ(ret, 1);
+ EXPECT_EQ(result[0].config_id, 180);
+ Maat_clean_status(&mid);
+
+ memset(result, 0, sizeof(result));
+ ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "fqdn_cat_id", strlen("fqdn_cat_id"));
+ ASSERT_EQ(ret, 0);
+
+ ret=Maat_scan_intval(g_feather, table_id, cat_id, result,4, &mid, 0);
+ EXPECT_EQ(ret, 1);
+ EXPECT_EQ(result[0].config_id, 180);
+
+ Maat_clean_status(&mid);
+
+ return;
+}
+
TEST(Hierarchy, VirtualWithVirtual)
{
#define Hierarchy_VirtualWithVirtual
@@ -1836,6 +1952,7 @@ TEST(Hierarchy, VirtualWithVirtual)
return;
}
+
TEST(Hierarchy, OneGroupInTwoVirtual)
{
#define Hierarchy_OneGroupInTwoVirtual
@@ -3110,9 +3227,9 @@ TEST_F(MaatCmdTest, PluginEXData)
return;
}
-TEST_F(MaatCmdTest, IPPluginEXData)
+TEST_F(MaatCmdTest, UpdateIPPlugin)
{
-#define ip_plugin_EX_data
+#define IP_Plugin_EX_data
Maat_feather_t feather=MaatCmdTest::_shared_feather;
int ret=0, i=0;
@@ -3178,7 +3295,7 @@ TEST_F(MaatCmdTest, IPPluginEXData)
ip_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL);
}
- ret=Maat_cmd_set_lines(feather, p_line , TEST_CMD_LINE_NUM, MAAT_OP_DEL);
+ ret=Maat_cmd_set_lines(feather, p_line, TEST_CMD_LINE_NUM, MAAT_OP_DEL);
EXPECT_GT(ret, 0);
usleep(WAIT_FOR_EFFECTIVE_US);
@@ -3188,6 +3305,75 @@ TEST_F(MaatCmdTest, IPPluginEXData)
return;
}
+
+TEST_F(MaatCmdTest, UpdateFQDNPlugin)
+{
+#define FQDN_Plugin_EX_data
+ Maat_feather_t feather=MaatCmdTest::_shared_feather;
+
+ int ret=0, i=0;
+ int table_id=0, fqdn_plugin_ex_data_counter=0;
+ const char* table_name="TEST_FQDN_PLUGIN_WITH_EXDATA";
+
+
+ const int TEST_CMD_LINE_NUM=5;
+ const struct Maat_cmd_line *p_line[TEST_CMD_LINE_NUM];
+ struct Maat_cmd_line line_rule[TEST_CMD_LINE_NUM];
+ const char* table_line[TEST_CMD_LINE_NUM]={
+ "201\t0\twww.example1.com\tcatid=1\t1",
+ "202\t1\t.example1.com\tcatid=1\t1",
+ "203\t0\tnews.example1.com\tcatid=2\t1",
+ "204\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1",
+ "205\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1"
+ };
+ table_id=Maat_table_register(feather, table_name);
+ ASSERT_GT(table_id, 0);
+ memset(&line_rule,0,sizeof(line_rule));
+ for(i=0;i<TEST_CMD_LINE_NUM;i++)
+ {
+ line_rule[i].label_id=0;
+ line_rule[i].rule_id=(int)Maat_cmd_incrby(feather,"TEST_PLUG_SEQ", 1);
+ line_rule[i].table_name=table_name;
+ line_rule[i].table_line=table_line[i];
+ line_rule[i].expire_after=0;
+ p_line[i]=line_rule+i;
+ }
+ ret=Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
+ EXPECT_GT(ret, 0);
+
+ usleep(WAIT_FOR_EFFECTIVE_US);
+
+ ret=Maat_fqdn_plugin_EX_register(feather, table_id,
+ fqdn_plugin_EX_new_cb,
+ fqdn_plugin_EX_free_cb,
+ fqdn_plugin_EX_dup_cb,
+ 0, &fqdn_plugin_ex_data_counter);
+
+ ASSERT_TRUE(ret>=0);
+ EXPECT_EQ(fqdn_plugin_ex_data_counter, 5);
+
+
+ struct fqdn_plugin_ud* result[4];
+
+ ret=Maat_fqdn_plugin_get_EX_data(g_feather, table_id, "r3---sn-i3belne6.example2.com", (void**)result, 4);
+ ASSERT_EQ(ret, 2);
+ for(i=0; i<ret; i++)
+ {
+ fqdn_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL);
+ }
+
+ ret=Maat_cmd_set_lines(feather, p_line+3, TEST_CMD_LINE_NUM-3, MAAT_OP_DEL);
+ EXPECT_GT(ret, 0);
+
+ usleep(WAIT_FOR_EFFECTIVE_US*5);
+ ret=Maat_fqdn_plugin_get_EX_data(feather, table_id, "r3---sn-i3belne6.example2.com", (void**)result, 4);
+ ASSERT_EQ(ret, 0);
+
+ return;
+}
+
+
+
#define TEST_HIT_PATH
TEST_F(MaatCmdTest, HitPath)
{