diff options
| -rw-r--r-- | common/include/tfe_cmsg.h | 2 | ||||
| -rw-r--r-- | common/src/tfe_scan.cpp | 51 | ||||
| -rw-r--r-- | plugin/business/pangu-http/src/edit_element.cpp | 104 | ||||
| -rw-r--r-- | plugin/business/pangu-http/src/pangu_http.cpp | 29 | ||||
| -rw-r--r-- | plugin/business/pangu-http/src/test_edit_element.cpp | 3 |
5 files changed, 156 insertions, 33 deletions
diff --git a/common/include/tfe_cmsg.h b/common/include/tfe_cmsg.h index bf0724b..5d60191 100644 --- a/common/include/tfe_cmsg.h +++ b/common/include/tfe_cmsg.h @@ -81,6 +81,8 @@ enum tfe_cmsg_tlv_type TFE_CMSG_DST_IP_LOCATION_PROVINE, // string max size 256 TFE_CMSG_SRC_IP_LOCATION_CITY, // string max size 256 TFE_CMSG_DST_IP_LOCATION_CITY, // string max size 256 + TFE_CMSG_SRC_IP_LOCATION_SUBDIVISION, + TFE_CMSG_DST_IP_LOCATION_SUBDIVISION, /* SSL ja3 fingerprint */ TFE_CMSG_SSL_CLIENT_JA3_FINGERPRINT, // string max size 32 diff --git a/common/src/tfe_scan.cpp b/common/src/tfe_scan.cpp index 05639f0..aa3feb5 100644 --- a/common/src/tfe_scan.cpp +++ b/common/src/tfe_scan.cpp @@ -134,6 +134,8 @@ int tfe_scan_ip_location(const struct tfe_stream *stream, struct Maat_rule_t *re char dst_provine[TFE_STRING_MAX] = {0}; char src_country[TFE_STRING_MAX] = {0}; char dst_country[TFE_STRING_MAX] = {0}; + char src_subdivsion[TFE_STRING_MAX] = {0}; + char dst_subdivsion[TFE_STRING_MAX] = {0}; struct tfe_cmsg *cmsg = tfe_stream_get0_cmsg(stream); if (cmsg != NULL) { @@ -167,13 +169,31 @@ int tfe_scan_ip_location(const struct tfe_stream *stream, struct Maat_rule_t *re { TFE_LOG_ERROR(logger, "fetch dst city from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info); } + scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_SRC_IP_LOCATION_SUBDIVISION, (unsigned char *)src_subdivsion, sizeof(src_subdivsion), &opt_out_size); + if (scan_ret != 0) + { + TFE_LOG_ERROR(logger, "fetch src subdivsion from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info); + } + scan_ret = tfe_cmsg_get_value(cmsg, TFE_CMSG_DST_IP_LOCATION_SUBDIVISION, (unsigned char *)dst_subdivsion, sizeof(dst_subdivsion), &opt_out_size); + if (scan_ret != 0) + { + TFE_LOG_ERROR(logger, "fetch dst subdivsion from cmsg failed, ret: %d addr: %s", scan_ret, stream->str_stream_info); + } } - TFE_LOG_DEBUG(logger, "fetch src country:%s provine:%s city:%s; dst country:%s provine:%s city:%s addr: %s", src_country, src_provine, src_city, dst_country, dst_provine, dst_city, stream->str_stream_info); + TFE_LOG_DEBUG(logger, "fetch src country:%s provine:%s city:%s; subdivsion:%s, dst country:%s provine:%s city:%s subdivsion:%s addr: %s", src_country, src_provine, src_city, src_subdivsion, dst_country, dst_provine, dst_city, dst_subdivsion, stream->str_stream_info); if (strlen(dst_country) || strlen(dst_city)) { memset(buff, 0, sizeof(buff)); - snprintf(buff, sizeof(buff), "%s.%s.", dst_country, dst_city); + + if(strlen(dst_subdivsion)) + { + snprintf(buff, sizeof(buff),"%s.%s.%s.%s.", dst_country, dst_provine, dst_city, dst_subdivsion); + } + else + { + snprintf(buff, sizeof(buff), "%s.%s.", dst_country, dst_city); + } scan_ret = Maat_full_scan_string(tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(TABLE_SECURITY_DESTINATION_LOCATION), CHARSET_GBK, buff, strlen(buff), result + hit_cnt + hit_cnt_ip, NULL, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip, @@ -190,13 +210,27 @@ int tfe_scan_ip_location(const struct tfe_stream *stream, struct Maat_rule_t *re buff, scan_ret, stream->str_stream_info); } memset(buff, 0, sizeof(buff)); - snprintf(buff, sizeof(buff), "%s,%s,%s", dst_city, dst_provine, dst_country); + if(strlen(dst_subdivsion)) + { + snprintf(buff, sizeof(buff), "%s,%s,%s,%s", dst_country, dst_provine, dst_city, dst_subdivsion); + } + else + { + snprintf(buff, sizeof(buff), "%s,%s,%s", dst_country, dst_provine, dst_city); + } *location_server = tfe_strdup(buff); } if (strlen(src_country) || strlen(src_city)) { memset(buff, 0, sizeof(buff)); - snprintf(buff, sizeof(buff), "%s.%s.", src_country, src_city); + if(strlen(src_subdivsion)) + { + snprintf(buff, sizeof(buff), "%s.%s.%s.%s.", src_country, src_provine, src_city, src_subdivsion); + } + else + { + snprintf(buff, sizeof(buff), "%s.%s.", src_country, src_city); + } scan_ret = Maat_full_scan_string(tfe_bussiness_resouce_get(STATIC_MAAT), tfe_bussiness_tableid_get(TABLE_SECURITY_SOURCE_LOCATION), CHARSET_GBK, buff, strlen(buff), result + hit_cnt + hit_cnt_ip, NULL, MAX_SCAN_RESULT - hit_cnt - hit_cnt_ip, @@ -214,7 +248,14 @@ int tfe_scan_ip_location(const struct tfe_stream *stream, struct Maat_rule_t *re } memset(buff, 0, sizeof(buff)); - snprintf(buff, sizeof(buff), "%s,%s,%s", src_city, src_provine, src_country); + if(strlen(src_subdivsion)) + { + snprintf(buff, sizeof(buff), "%s,%s,%s,%s", src_country, src_provine, src_city, src_subdivsion); + } + else + { + snprintf(buff, sizeof(buff), "%s,%s,%s", src_country, src_provine, src_city); + } *location_client = tfe_strdup(buff); } diff --git a/plugin/business/pangu-http/src/edit_element.cpp b/plugin/business/pangu-http/src/edit_element.cpp index 182b88b..ed686e2 100644 --- a/plugin/business/pangu-http/src/edit_element.cpp +++ b/plugin/business/pangu-http/src/edit_element.cpp @@ -5,6 +5,11 @@ #include <stdlib.h> #include <assert.h> +#if 0 +#define PCRE2_CODE_UNIT_WIDTH 8 +#include <pcre2.h> +#endif + #include <libxml/tree.h> #include <libxml/xmlsave.h> #include <libxml/HTMLparser.h> @@ -13,7 +18,7 @@ #include "edit_element.h" int cjson_element_foreach(cJSON *a, int *depth, int *step, int *step_level, char **node, const struct edit_element_rule * rules, int *match_num, int loop); -static void html_node_list(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match); +static void html_node_list(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match,size_t mark_tag); size_t parse_json_output_unformatted(const char * in, size_t in_sz, const struct edit_element_rule * rules, char** out); enum search_scope scope_name_to_id(const char * name) @@ -52,6 +57,36 @@ int match_start_indicator(xmlNodePtr parent, char * start_indicator) return 0; } +#if 0 +int match_string(const char * in, size_t in_sz, char *contained_keyword) +{ + assert(strlen(contained_keyword) != 0); + + int error=0; + PCRE2_SIZE erroffset=0; + + const PCRE2_SPTR pattern = (PCRE2_SPTR)contained_keyword; + uint32_t pcre2_options = PCRE2_UTF; + + pcre2_code *re = pcre2_compile(pattern, strlen(contained_keyword), pcre2_options, &error, &erroffset, 0); + if(!re) + { + return 0; + } + + pcre2_match_data *match_data = pcre2_match_data_create_from_pattern(re, NULL); + int rc = 0; + const PCRE2_SPTR subject = (PCRE2_SPTR)in; + + rc = pcre2_match(re, subject, in_sz, 0, 0, match_data, NULL); + + pcre2_match_data_free(match_data); + pcre2_code_free(re); + + return rc; +} +#endif + int construct_cjson_by_treatment(cJSON *a_element, char **node, int *step, int *step_array_level, const struct edit_element_rule * rules) { const char *element_treatment=rules->element_treatment; @@ -151,6 +186,12 @@ int construct_html_by_treatment(const struct edit_element_rule * rules, xmlNodeP } } +#if 0 + if(match_string((char *)node->content, strlen((char *)node->content), rules->contained_keyword) < 0) + { + return 0; + } +#endif if(strcasestr((char *)node->content, rules->contained_keyword) == NULL) { return 0; @@ -201,12 +242,8 @@ int construct_html_by_treatment(const struct edit_element_rule * rules, xmlNodeP break; } - if(*n_parent < 16) - { - parent_array[*n_parent] = parent; - *n_parent = *n_parent+1; - } - + parent_array[0] = parent; + *n_parent = *n_parent+1; *match =1; break; } @@ -281,6 +318,9 @@ int cjson_dump_string(cJSON *a, int *depth, const struct edit_element_rule * rul { int xret=0; +#if 0 + if((a->valuestring != NULL) && (match_string(a->valuestring, strlen(a->valuestring), rules->contained_keyword) > 0)) +#endif if((a->valuestring != NULL) && strcasestr(a->valuestring, rules->contained_keyword)) { if(*depth != -1) @@ -345,20 +385,20 @@ static void html_namespace_list(xmlNsPtr ns) } } -static void html_attr_list(const struct edit_element_rule * rules, xmlAttrPtr attr, xmlNodePtr *parent_array, size_t *n_parent, int *match) +static void html_attr_list(const struct edit_element_rule * rules, xmlAttrPtr attr, xmlNodePtr *parent_array, size_t *n_parent, int *match, size_t mark_tag) { while (attr != NULL) { if (attr->children != NULL) { - html_node_list(rules, attr->children, parent_array, n_parent, match); + html_node_list(rules, attr->children, parent_array, n_parent, match, mark_tag); } attr = attr->next; } } -static void html_dump_one_node(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match) +static void html_dump_one_node(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match, size_t mark_tag) { switch (node->type) { @@ -385,7 +425,7 @@ static void html_dump_one_node(const struct edit_element_rule * rules, xmlNodePt if ((node->type == XML_ELEMENT_NODE) && (node->properties != NULL)) { - html_attr_list(rules, node->properties, parent_array, n_parent, match); + html_attr_list(rules, node->properties, parent_array, n_parent, match, mark_tag); } if (node->type != XML_ENTITY_REF_NODE) @@ -397,29 +437,39 @@ static void html_dump_one_node(const struct edit_element_rule * rules, xmlNodePt } } -static void html_dump_node(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match) +static void html_dump_node(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match, size_t mark_tag) { - html_dump_one_node(rules, node, parent_array, n_parent, match); + html_dump_one_node(rules, node, parent_array, n_parent, match, mark_tag); + if(*match == 1 && mark_tag == 0) + { + return; + } + if ((node->type != XML_NAMESPACE_DECL) && (node->children != NULL) && (node->type != XML_ENTITY_REF_NODE)) { - html_node_list(rules, node->children, parent_array, n_parent, match); + html_node_list(rules, node->children, parent_array, n_parent, match, mark_tag); } } -static void html_node_list(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match) +static void html_node_list(const struct edit_element_rule * rules, xmlNodePtr node, xmlNodePtr *parent_array, size_t *n_parent, int *match, size_t mark_tag) { while (node != NULL) { - html_dump_node(rules, node, parent_array, n_parent, match); + html_dump_node(rules, node, parent_array, n_parent, match, mark_tag); + if(*match == 1 && mark_tag == 0) + { + break; + } + node = node->next; } } -static void html_element_foreach(const struct edit_element_rule * rules, xmlDocPtr doc, xmlNodePtr *parent_array, size_t *n_parent, int *match) +static void html_element_foreach(const struct edit_element_rule * rules, xmlDocPtr doc, xmlNodePtr *parent_array, size_t *n_parent, int *match, size_t mark_tag) { if (((doc->type == XML_DOCUMENT_NODE) || (doc->type == XML_HTML_DOCUMENT_NODE)) && (doc->children != NULL)) { - html_node_list(rules, doc->children, parent_array, n_parent, match); + html_node_list(rules, doc->children, parent_array, n_parent, match, mark_tag); } } @@ -476,6 +526,11 @@ size_t parse_json_output_unformatted(const char * in, size_t in_sz, const struct } } + if(match == 0) + { + goto finish; + } + new_out = cJSON_PrintUnformatted(interator); if(new_out!=NULL) { @@ -666,9 +721,8 @@ size_t format_input_html(const char * in, size_t in_sz, const struct edit_elemen goto finish; } - /*When the node has inclusion relation, libxml2 is not null when deleted - So multiple loops delete **/ - html_element_foreach(rules, doc, parent_array, &n_parent, &match); + /*Delete all at once, valgrind is invalid read **/ + html_element_foreach(rules, doc, parent_array, &n_parent, &match, 1); if(match != 1) { goto finish; @@ -677,13 +731,15 @@ size_t format_input_html(const char * in, size_t in_sz, const struct edit_elemen n_parent_peer = n_parent; element_treatment=rules->element_treatment; + /*When the node has inclusion relation, libxml2 is not null when deleted + So multiple loops delete **/ if(element_treatment != NULL && !strcasecmp(element_treatment, "remove")) { for(i=0; i < (int)n_parent_peer; i++) { match =0; n_parent = 0; - html_element_foreach(rules, doc, parent_array, &n_parent, &match); - if(match == 1) + html_element_foreach(rules, doc, parent_array, &n_parent, &match, 0); + if(match == 1 && n_parent > 0) { xmlUnlinkNode(parent_array[0]); xmlFreeNode(parent_array[0]); @@ -721,7 +777,7 @@ size_t format_html_file_type(const char * interator, size_t interator_sz, const { size_t output_size=0; - if(interator[0] == '{') + if((interator[0] == '{') || (interator[0] == '[')) { output_size = format_multidelete_json_type(interator, interator_sz, rule, new_out); } diff --git a/plugin/business/pangu-http/src/pangu_http.cpp b/plugin/business/pangu-http/src/pangu_http.cpp index 37ead64..52e3229 100644 --- a/plugin/business/pangu-http/src/pangu_http.cpp +++ b/plugin/business/pangu-http/src/pangu_http.cpp @@ -2216,6 +2216,29 @@ static void http_manipulate(const struct tfe_stream * stream, const struct tfe_h return; } +static int get_fqdn_len(char *str_host) +{ + char *p=NULL; int fqdn_len=0; + + if(str_host == NULL) + { + goto finish; + } + + p=index(str_host, ':'); + if(p==NULL) + { + fqdn_len=strlen(str_host); + } + else + { + fqdn_len=p-str_host; + } + +finish: + return fqdn_len; +} + enum pangu_action http_scan(const struct tfe_http_session * session, enum tfe_http_event events, const unsigned char * body_frag, size_t frag_size, struct pangu_http_ctx * ctx, const struct tfe_stream * stream) { @@ -2230,10 +2253,10 @@ enum pangu_action http_scan(const struct tfe_http_session * session, enum tfe_ht if (events & EV_HTTP_REQ_HDR) { - const char *str_host = session->req->req_spec.host; - if (str_host != NULL) + char *str_host = (char *)session->req->req_spec.host; + int str_host_length = get_fqdn_len(str_host); + if (str_host != NULL && str_host_length != 0) { - int str_host_length = (int) (strlen(session->req->req_spec.host)); scan_ret = Maat_full_scan_string(g_pangu_rt->maat, g_pangu_rt->scan_table_id[PXY_CTRL_HTTP_FQDN], CHARSET_UTF8, str_host, str_host_length, result + hit_cnt, NULL, MAX_SCAN_RESULT - hit_cnt, &(ctx->scan_mid), ctx->thread_id); if (scan_ret > 0) diff --git a/plugin/business/pangu-http/src/test_edit_element.cpp b/plugin/business/pangu-http/src/test_edit_element.cpp index a16fc4b..07b2eb8 100644 --- a/plugin/business/pangu-http/src/test_edit_element.cpp +++ b/plugin/business/pangu-http/src/test_edit_element.cpp @@ -101,7 +101,7 @@ TEST(EditElement, Cjson_Whole_mark_Simple) free(output); } -TEST(EditElement, Libxml_Whole_Remove_Facebook) +TEST(EditElement, Libxml_Inside_Remove_Facebook) { char* output=NULL; size_t output_sz=0,input_len=0; @@ -130,6 +130,7 @@ TEST(EditElement, Libxml_Whole_Remove_Facebook) EXPECT_TRUE(output_sz>0); EXPECT_TRUE(NULL==strstr(output, "_2t-a _4pmj _2t-d")); + free(input); free(output); } |