diff options
| author | liuwentan <[email protected]> | 2024-01-26 17:21:36 +0800 |
|---|---|---|
| committer | liuwentan <[email protected]> | 2024-02-01 19:43:09 +0800 |
| commit | 06af77fb35a55754768f91fbf1363e2dd7906683 (patch) | |
| tree | fd09d0c99e479bd3a18df4e4ef346702334fbb45 | |
| parent | 889f024485870e988d6185affff31306843b566a (diff) | |
[HTTP_DECODER]optimize http header cache
| -rw-r--r-- | src/http_decoder/http_content_decompress.c | 4 | ||||
| -rw-r--r-- | src/http_decoder/http_decoder.c | 163 | ||||
| -rw-r--r-- | src/http_decoder/http_decoder_half.c | 75 | ||||
| -rw-r--r-- | src/http_decoder/http_decoder_inc.h | 26 | ||||
| -rw-r--r-- | src/http_decoder/http_decoder_string.c | 120 | ||||
| -rw-r--r-- | src/http_decoder/http_decoder_string.h | 6 | ||||
| -rw-r--r-- | src/http_decoder/http_decoder_table.c | 80 | ||||
| -rw-r--r-- | src/http_decoder/http_decoder_table.h | 15 | ||||
| -rw-r--r-- | test/http_decoder/CMakeLists.txt | 3 | ||||
| -rw-r--r-- | test/http_decoder/http_decoder_gtest.cpp | 20 | ||||
| -rw-r--r-- | test/http_decoder/test_result_json/http_get_long_cookie.json | 6 | ||||
| -rw-r--r-- | test/http_decoder/test_result_json/http_headers_exceed_maximum.json | 10 | ||||
| -rw-r--r-- | test/http_decoder/test_result_json/http_req_1byte_sliding_window.json | 32 | ||||
| -rw-r--r-- | test/http_decoder/test_result_json/http_res_1byte_sliding_window.json | 18 | ||||
| -rw-r--r-- | test/http_decoder/test_result_json/http_trans_pipeline.json | 56 |
15 files changed, 471 insertions, 163 deletions
diff --git a/src/http_decoder/http_content_decompress.c b/src/http_decoder/http_content_decompress.c index b887487..c12cdf4 100644 --- a/src/http_decoder/http_content_decompress.c +++ b/src/http_decoder/http_content_decompress.c @@ -221,8 +221,8 @@ http_content_decompress_write_br(struct http_content_decompress *decompress, } } - if (ret == BROTLI_DECODER_RESULT_SUCCESS - || ret == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { + if (ret == BROTLI_DECODER_RESULT_SUCCESS || + ret == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { return 0; } diff --git a/src/http_decoder/http_decoder.c b/src/http_decoder/http_decoder.c index 523253b..1ce4b86 100644 --- a/src/http_decoder/http_decoder.c +++ b/src/http_decoder/http_decoder.c @@ -55,10 +55,31 @@ struct http_decoder_context { int fs_incoming_bytes_id; int fs_incoming_pkts_id; int fs_incoming_trans_id; + int fs_err_pkts_id; struct fieldstat_easy *fse; struct stellar *st; }; +struct http_message * +http_message_new(enum http_message_type type, void *data) +{ + struct http_message *msg = CALLOC(struct http_message, 1); + + msg->type = type; + msg->data = data; + + return msg; +} + +static void http_message_free(void *http_msg, void *cb_arg) +{ + if (NULL == http_msg) { + return; + } + + FREE(http_msg); +} + static void http_event_handler(enum http_event event, struct http_decoder_half_data **data, struct http_event_context *ev_ctx) @@ -94,23 +115,17 @@ static void http_event_handler(enum http_event event, *data = half_data; break; case HTTP_EVENT_REQ_LINE: - msg = CALLOC(struct http_message, 1); - msg->type = HTTP_MESSAGE_REQ_LINE; - msg->data = *data; + msg = http_message_new(HTTP_MESSAGE_REQ_LINE, *data); session_mq_publish_message(ev_ctx->ref_session, ev_ctx->topic_id, msg); break; case HTTP_EVENT_REQ_HDR_END: - msg = CALLOC(struct http_message, 1); - msg->type = HTTP_MESSAGE_REQ_HEADER; - msg->data = *data; + msg = http_message_new(HTTP_MESSAGE_REQ_HEADER, *data); session_mq_publish_message(ev_ctx->ref_session, ev_ctx->topic_id, msg); break; case HTTP_EVENT_REQ_BODY_BEGIN: break; case HTTP_EVENT_REQ_BODY_DATA: - msg = CALLOC(struct http_message, 1); - msg->type = HTTP_MESSAGE_REQ_BODY; - msg->data = *data; + msg = http_message_new(HTTP_MESSAGE_REQ_BODY, *data); session_mq_publish_message(ev_ctx->ref_session, ev_ctx->topic_id, msg); break; case HTTP_EVENT_REQ_BODY_END: @@ -146,25 +161,19 @@ static void http_event_handler(enum http_event event, *data = half_data; break; case HTTP_EVENT_RES_LINE: - msg = CALLOC(struct http_message, 1); - msg->type = HTTP_MESSAGE_RES_LINE; - msg->data = *data; + msg = http_message_new(HTTP_MESSAGE_RES_LINE, *data); session_mq_publish_message(ev_ctx->ref_session, ev_ctx->topic_id, msg); break; case HTTP_EVENT_RES_HDR: break; case HTTP_EVENT_RES_HDR_END: - msg = CALLOC(struct http_message, 1); - msg->type = HTTP_MESSAGE_RES_HEADER; - msg->data = *data; + msg = http_message_new(HTTP_MESSAGE_RES_HEADER, *data); session_mq_publish_message(ev_ctx->ref_session, ev_ctx->topic_id, msg); break; case HTTP_EVENT_RES_BODY_BEGIN: break; case HTTP_EVENT_RES_BODY_DATA: - msg = CALLOC(struct http_message, 1); - msg->type = HTTP_MESSAGE_RES_BODY; - msg->data = *data; + msg = http_message_new(HTTP_MESSAGE_RES_BODY, *data); session_mq_publish_message(ev_ctx->ref_session, ev_ctx->topic_id, msg); break; case HTTP_EVENT_RES_BODY_END: @@ -320,7 +329,9 @@ int http_decoder_entry(struct session *sess, int events, return -1; } + int thread_id = session_get_current_thread_id(sess); struct http_decoder_half *cur_half = NULL; + if (dir == PACKET_DIRECTION_C2S) { cur_half = ex_data->decoder->c2s_half; } else { @@ -328,38 +339,29 @@ int http_decoder_entry(struct session *sess, int events, } http_decoder_half_reinit(cur_half, decoder_ctx->topic_id, ex_data->queue, sess); - http_decoder_half_parse(cur_half, payload, payload_len); - - int thread_id = session_get_current_thread_id(sess); + ret = http_decoder_half_parse(cur_half, payload, payload_len); + if (ret < 0) { + // fieldstat_easy_counter_incrby(decoder_ctx->fse, thread_id, + // decoder_ctx->fs_err_pkts_id, + // NULL, 0, 1); + } - fieldstat_easy_counter_incrby(decoder_ctx->fse, thread_id, - decoder_ctx->fs_incoming_bytes_id, - NULL, 0, payload_len); + // fieldstat_easy_counter_incrby(decoder_ctx->fse, thread_id, + // decoder_ctx->fs_incoming_bytes_id, + // NULL, 0, payload_len); - fieldstat_easy_counter_incrby(decoder_ctx->fse, thread_id, - decoder_ctx->fs_incoming_pkts_id, - NULL, 0, 1); + // fieldstat_easy_counter_incrby(decoder_ctx->fse, thread_id, + // decoder_ctx->fs_incoming_pkts_id, + // NULL, 0, 1); - long long trans_cnt = http_decoder_half_trans_count(cur_half); - fieldstat_easy_counter_incrby(decoder_ctx->fse, thread_id, - decoder_ctx->fs_incoming_trans_id, - NULL, 0, trans_cnt); + // long long trans_cnt = http_decoder_half_trans_count(cur_half); + // fieldstat_easy_counter_incrby(decoder_ctx->fse, thread_id, + // decoder_ctx->fs_incoming_trans_id, + // NULL, 0, trans_cnt); return 0; } -static void http_message_free(void *msg, void *cb_arg) -{ - if (NULL == msg) { - return; - } - - struct http_message *message = (struct http_message *)msg; - message->data = NULL; //don't have memory's ownership - - FREE(message); -} - static void _http_decoder_context_free(struct http_decoder_context *ctx) { if (NULL == ctx) { @@ -391,65 +393,84 @@ static void http_decoder_ex_data_free(struct session *s, int idx, } #define FS_OUTPUT_INTERVAL_S 1 -void *http_decoder_init(struct stellar *st) +static int http_decoder_fs_init(struct http_decoder_context *ctx, int thread_num) { - struct http_decoder_context *ctx = CALLOC(struct http_decoder_context, 1); - - ctx->st = st; - ctx->ex_data_idx = stellar_session_get_ex_new_index(st, "HTTP_DECODER", - http_decoder_ex_data_free, - NULL); - - int plugin_id = stellar_plugin_register(st, SESS_EV_TCP|SESS_EV_CLOSING, - http_decoder_entry, ctx); - if (plugin_id >= 0) { - ctx->plugin_id = plugin_id; - } - - int topic_id = session_mq_get_topic_id(st, http_decoder_topic); - if (topic_id < 0) { - topic_id = session_mq_create_topic(st, http_decoder_topic, - http_message_free, NULL); - } - ctx->topic_id = topic_id; - - int thread_num = stellar_get_worker_thread_num(st); - ctx->fse = fieldstat_easy_new(thread_num, "http_decoder_statistics", NULL, 0); if (NULL == ctx->fse) { fprintf(stderr, "fieldstat_easy_new failed."); - goto failed; + return -1; } ctx->fs_incoming_bytes_id = fieldstat_easy_register_counter(ctx->fse, "incoming_bytes"); if (ctx->fs_incoming_bytes_id < 0) { fprintf(stderr, "fieldstat_easy_register_counter incoming_bytes failed."); - goto failed; + return -1; } ctx->fs_incoming_trans_id = fieldstat_easy_register_counter(ctx->fse, "incoming_trans"); if (ctx->fs_incoming_trans_id < 0) { fprintf(stderr, "fieldstat_easy_register_counter incoming_trans failed."); - goto failed; + return -1; } ctx->fs_incoming_pkts_id = fieldstat_easy_register_counter(ctx->fse, "incoming_pkts"); if (ctx->fs_incoming_pkts_id < 0) { fprintf(stderr, "fieldstat_easy_register_counter incoming_pkts failed."); - goto failed; + return -1; + } + + ctx->fs_err_pkts_id = fieldstat_easy_register_counter(ctx->fse, "err_pkts"); + if (ctx->fs_err_pkts_id < 0) { + fprintf(stderr, "fieldstat_easy_register_counter err_pkts failed."); + return -1; } int ret = fieldstat_easy_enable_auto_output(ctx->fse, fs_file_name, FS_OUTPUT_INTERVAL_S); if (ret < 0) { fprintf(stderr, "fieldstat_easy_enable_auto_output failed."); - goto failed; + return -1; } + sleep(1); + return 0; +} + +void *http_decoder_init(struct stellar *st) +{ + struct http_decoder_context *ctx = CALLOC(struct http_decoder_context, 1); + + ctx->st = st; + ctx->ex_data_idx = stellar_session_get_ex_new_index(st, "HTTP_DECODER", + http_decoder_ex_data_free, + NULL); + + int plugin_id = stellar_plugin_register(st, SESS_EV_TCP|SESS_EV_CLOSING, + http_decoder_entry, ctx); + if (plugin_id < 0) { + return NULL; + } + + ctx->plugin_id = plugin_id; + + int topic_id = session_mq_get_topic_id(st, http_decoder_topic); + if (topic_id < 0) { + topic_id = session_mq_create_topic(st, http_decoder_topic, + http_message_free, NULL); + } + + ctx->topic_id = topic_id; + + int thread_num = stellar_get_worker_thread_num(st); + + if (http_decoder_fs_init(ctx, thread_num) < 0) { + goto failed; + } + printf("http_decoder_init: ex_data_idx:%d, plugin_id:%d, topic_id:%d\n", ctx->ex_data_idx, ctx->plugin_id, ctx->topic_id); diff --git a/src/http_decoder/http_decoder_half.c b/src/http_decoder/http_decoder_half.c index d97cd25..cc13d8e 100644 --- a/src/http_decoder/http_decoder_half.c +++ b/src/http_decoder/http_decoder_half.c @@ -10,9 +10,13 @@ #include <assert.h> #include <stdio.h> +#include <string.h> #include "stellar/utils.h" +#include "stellar/session_mq.h" #include "llhttp.h" +#include "http_decoder.h" +#include "http_decoder_inc.h" #include "http_decoder_utils.h" #include "http_decoder_half.h" #include "http_decoder_table.h" @@ -43,6 +47,7 @@ struct http_decoder_half { http_event_cb *http_ev_cb; struct http_event_context *http_ev_ctx; long long trans_counter; + long long err_counter; }; // #define HTTP_DECODER_DEBUG @@ -194,6 +199,7 @@ static int on_method_complete(llhttp_t *http) STRING_STATE_REFER) { http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_METHOD); } + http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_METHOD); return 0; @@ -226,6 +232,7 @@ static int on_uri_complete(llhttp_t *http) STRING_STATE_REFER) { http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_URI); } + http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_URI); return 0; @@ -258,6 +265,7 @@ static int on_version_complete(llhttp_t *http) STRING_STATE_REFER) { http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_VERSION); } + http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_VERSION); half->ref_data->major_version = llhttp_get_http_major(&half->parser); @@ -299,14 +307,16 @@ static int on_status_complete(llhttp_t *http) if (http_decoder_table_state(half->ref_data->table, HTTP_ITEM_STATUS) == STRING_STATE_REFER) { http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_STATUS); - } - http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_STATUS); + } + http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_STATUS); half->ref_data->status_code = llhttp_get_status_code(&half->parser); - half->event = HTTP_EVENT_RES_LINE; - if (half->http_ev_cb != NULL) { - half->http_ev_cb(half->event, &half->ref_data, half->http_ev_ctx); + if (half->parser.type == HTTP_RESPONSE) { + half->event = HTTP_EVENT_RES_LINE; + if (half->http_ev_cb != NULL) { + half->http_ev_cb(half->event, &half->ref_data, half->http_ev_ctx); + } } return 0; @@ -335,10 +345,6 @@ static int on_header_field_complete(llhttp_t *http) container_of(http, struct http_decoder_half, parser); assert(half); - if (http_decoder_table_state(half->ref_data->table, HTTP_ITEM_HDRKEY) == - STRING_STATE_REFER) { - http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_HDRKEY); - } http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_HDRKEY); return 0; @@ -367,10 +373,11 @@ static int on_header_value_complete(llhttp_t *http) container_of(http, struct http_decoder_half, parser); assert(half); - if (http_decoder_table_state(half->ref_data->table, HTTP_ITEM_HDRVAL) == - STRING_STATE_REFER) { - http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_HDRVAL); + if (http_decoder_table_state(half->ref_data->table, HTTP_ITEM_HDRKEY) == + STRING_STATE_CACHE) { + http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_HDRKEY); } + http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_HDRVAL); if (half->ref_data->content_encoding == HTTP_CONTENT_ENCODING_NONE) { @@ -379,10 +386,9 @@ static int on_header_value_complete(llhttp_t *http) if (http_decoder_table_get_header(half->ref_data->table, &key, &http_hdr, 1) == 1) { - char *str = safe_dup(http_hdr.val.str, http_hdr.val.str_len); - half->ref_data->content_encoding = - http_content_encoding_str2int(str); - FREE(str); + char encoding_str[16] = {0}; + memcpy(encoding_str, http_hdr.val.str, http_hdr.val.str_len); + half->ref_data->content_encoding = http_content_encoding_str2int(encoding_str); } } @@ -476,8 +482,8 @@ static int on_body(llhttp_t *http, const char *at, size_t length) http_decoder_table_reset(half->ref_data->table, HTTP_ITEM_BODY); } - http_decoder_table_refer(half->ref_data->table, HTTP_ITEM_BODY, at, - length); + http_decoder_table_refer(half->ref_data->table, HTTP_ITEM_BODY, + at, length); http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_BODY); } @@ -510,7 +516,7 @@ static void http_decoder_half_init(struct http_decoder_half *half, llhttp_settings_init(&half->settings); llhttp_init(&half->parser, type, &half->settings); - + half->settings.on_message_begin = on_message_begin; half->settings.on_message_complete = on_message_complete; half->settings.on_reset = on_reset; @@ -578,11 +584,35 @@ void http_decoder_half_reinit(struct http_decoder_half *half, int topic_id, { assert(half != NULL); + if (half->ref_data != NULL) { + http_decoder_table_reinit(half->ref_data->table); + } + half->http_ev_ctx->topic_id = topic_id; half->http_ev_ctx->ref_session = sess; half->http_ev_ctx->ref_queue = queue; } +static void publish_message_for_parsed_header(struct http_decoder_half *half) +{ + if (http_decoder_table_has_parsed_header(half->ref_data->table) <= 0) { + return; + } + + // publish complete kv-header message + struct http_message *msg = NULL; + if (half->parser.type == HTTP_REQUEST) { + msg = http_message_new(HTTP_MESSAGE_REQ_HEADER, half->ref_data); + session_mq_publish_message(half->http_ev_ctx->ref_session, + half->http_ev_ctx->topic_id, msg); + } else { + // http response + msg = http_message_new(HTTP_MESSAGE_RES_HEADER, half->ref_data); + session_mq_publish_message(half->http_ev_ctx->ref_session, + half->http_ev_ctx->topic_id, msg); + } +} + int http_decoder_half_parse(struct http_decoder_half *half, const char *data, size_t data_len) { @@ -643,11 +673,18 @@ int http_decoder_half_parse(struct http_decoder_half *half, if (http_decoder_table_state(half->ref_data->table, HTTP_ITEM_HDRKEY) == STRING_STATE_REFER) { + publish_message_for_parsed_header(half); http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_HDRKEY); } if (http_decoder_table_state(half->ref_data->table, HTTP_ITEM_HDRVAL) == STRING_STATE_REFER) { + publish_message_for_parsed_header(half); + + /* Header key should have been committed + If it's not cached, cache it for next packet to use + */ + http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_HDRKEY); http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_HDRVAL); } diff --git a/src/http_decoder/http_decoder_inc.h b/src/http_decoder/http_decoder_inc.h new file mode 100644 index 0000000..fd39d6e --- /dev/null +++ b/src/http_decoder/http_decoder_inc.h @@ -0,0 +1,26 @@ +/* +********************************************************************************************** +* File: http_decoder_inc.h +* Description: +* Authors: Liu WenTan <[email protected]> +* Date: 2024-01-10 +* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _HTTP_DECODER_INC_H_ +#define _HTTP_DECODER_INC_H_ + +#ifdef __cplusplus +extern "C" +{ +#endif + +struct http_message; +struct http_message *http_message_new(enum http_message_type type, void *data); + +#ifdef __cplusplus +} +#endif + +#endif
\ No newline at end of file diff --git a/src/http_decoder/http_decoder_string.c b/src/http_decoder/http_decoder_string.c index cfaeb4c..5414c5b 100644 --- a/src/http_decoder/http_decoder_string.c +++ b/src/http_decoder/http_decoder_string.c @@ -59,32 +59,83 @@ void http_decoder_string_refer(struct http_decoder_string *rstr, rstr->state = STRING_STATE_REFER; } -void http_decoder_string_cache(struct http_decoder_string *rstr) +static void string_refer2cache(struct http_decoder_string *rstr) { - if (NULL == rstr) { + if (0 == rstr->refer.str_len) { return; } - size_t length = 0; + if (rstr->cache.str_len >= rstr->max_cache_size) { + return; + } - switch (rstr->state) { - case STRING_STATE_REFER: - if (rstr->refer.str_len > 0) { - if (NULL == rstr->cache.str) { - length = rstr->refer.str_len + 1; - rstr->cache.str = CALLOC(char, length); - } else { - length = rstr->cache.str_len + rstr->refer.str_len + 2; - rstr->cache.str = REALLOC(char, rstr->cache.str, length); - } + size_t length = rstr->cache.str_len + rstr->refer.str_len; + if (length > rstr->max_cache_size) { + length = rstr->max_cache_size; + } + + if (NULL == rstr->cache.str) { + rstr->cache.str = CALLOC(char, length + 1); + memcpy(rstr->cache.str, rstr->refer.str, length); + } else { + rstr->cache.str = REALLOC(char, rstr->cache.str, length + 1); + memcpy(rstr->cache.str + rstr->cache.str_len, rstr->refer.str, + (length - rstr->cache.str_len)); + } + + rstr->cache.str_len = length; + + rstr->refer.str = NULL; + rstr->refer.str_len = 0; +} + +static void string_commit2cache(struct http_decoder_string *rstr) +{ + if (rstr->cache.str_len == rstr->commit.str_len && + rstr->cache.str == rstr->commit.str) { - memcpy(rstr->cache.str + rstr->cache.str_len, rstr->refer.str, - rstr->refer.str_len); - rstr->cache.str_len += rstr->refer.str_len; + rstr->commit.str = NULL; + rstr->commit.str_len = 0; + return; + } - rstr->refer.str = NULL; - rstr->refer.str_len = 0; + //Only http header key need to backward to cache + size_t length = 0; + if (rstr->commit.str_len > rstr->max_cache_size) { + length = rstr->max_cache_size; + } else { + length = rstr->commit.str_len; + } + + if (length > 0) { + if (NULL == rstr->cache.str) { + rstr->cache.str = CALLOC(char, length + 1); + } else { + abort(); } + memcpy(rstr->cache.str, rstr->commit.str, length); + rstr->cache.str_len = length; + + rstr->commit.str = NULL; + rstr->commit.str_len = 0; + } +} + +void http_decoder_string_cache(struct http_decoder_string *rstr) +{ + if (NULL == rstr) { + return; + } + + switch (rstr->state) { + case STRING_STATE_REFER: + string_refer2cache(rstr); + break; + case STRING_STATE_CACHE: + break; + case STRING_STATE_COMMIT: + //commit backward to cache + string_commit2cache(rstr); break; default: abort(); @@ -149,6 +200,39 @@ void http_decoder_string_reset(struct http_decoder_string *rstr) rstr->state = STRING_STATE_INIT; } + +void http_decoder_string_init(struct http_decoder_string *rstr, + size_t max_cache_size) +{ + rstr->max_cache_size = max_cache_size; +} + +void http_decoder_string_reinit(struct http_decoder_string *rstr) +{ + if (rstr->state == STRING_STATE_CACHE) { + return; + } + + if (rstr->state == STRING_STATE_COMMIT && + rstr->cache.str == rstr->commit.str && + rstr->cache.str_len == rstr->commit.str_len) { + return; + } + + if (rstr->cache.str != NULL) { + FREE(rstr->cache.str); + rstr->cache.str_len = 0; + } + + rstr->refer.str = NULL; + rstr->refer.str_len = 0; + + rstr->commit.str = NULL; + rstr->commit.str_len = 0; + + rstr->state = STRING_STATE_INIT; +} + enum string_state http_decoder_string_state(struct http_decoder_string *rstr) { return rstr->state; diff --git a/src/http_decoder/http_decoder_string.h b/src/http_decoder/http_decoder_string.h index 804a7ff..f9d81dd 100644 --- a/src/http_decoder/http_decoder_string.h +++ b/src/http_decoder/http_decoder_string.h @@ -65,6 +65,7 @@ struct http_decoder_string { struct hstring commit; enum string_state state; + size_t max_cache_size; }; void http_decoder_string_refer(struct http_decoder_string *rstr, @@ -76,6 +77,11 @@ void http_decoder_string_commit(struct http_decoder_string *rstr); void http_decoder_string_reset(struct http_decoder_string *rstr); +void http_decoder_string_init(struct http_decoder_string *rstr, + size_t max_cache_size); + +void http_decoder_string_reinit(struct http_decoder_string *rstr); + enum string_state http_decoder_string_state(struct http_decoder_string *rstr); int http_decoder_string_get(struct http_decoder_string *rstr, struct hstring *out); diff --git a/src/http_decoder/http_decoder_table.c b/src/http_decoder/http_decoder_table.c index 300994d..ac12296 100644 --- a/src/http_decoder/http_decoder_table.c +++ b/src/http_decoder/http_decoder_table.c @@ -19,7 +19,12 @@ #include "stellar/utils.h" #define MAX_HEADER_SIZE 16 - +#define MAX_URI_CACHE_SIZE 1024 +#define MAX_STATUS_CACHE_SIZE 32 +#define MAX_METHOD_CACHE_SIZE 8 +#define MAX_VERSION_CACHE_SIZE 4 +#define MAX_HEADER_KEY_CACHE_SIZE 128 +#define MAX_HEADER_VALUE_CACHE_SIZE 1024 struct http_decoder_header { struct http_decoder_string key; @@ -39,15 +44,39 @@ struct http_decoder_table { struct http_decoder_header *headers; }; +static void http_decoder_table_init(struct http_decoder_table *table) +{ + if (NULL == table) { + return; + } + + struct http_decoder_header *header = NULL; + assert(table); + + http_decoder_string_init(&table->uri, MAX_URI_CACHE_SIZE); + http_decoder_string_init(&table->status, MAX_STATUS_CACHE_SIZE); + http_decoder_string_init(&table->method, MAX_METHOD_CACHE_SIZE); + http_decoder_string_init(&table->version, MAX_METHOD_CACHE_SIZE); + + for (size_t i = 0; i < table->header_size; i++) { + header = &table->headers[i]; + http_decoder_string_init(&header->key, MAX_HEADER_KEY_CACHE_SIZE); + http_decoder_string_init(&header->val, MAX_HEADER_VALUE_CACHE_SIZE); + } + + http_decoder_string_init(&table->body, 0); +} + struct http_decoder_table *http_decoder_table_new() { struct http_decoder_table *table = CALLOC(struct http_decoder_table, 1); assert(table); - table->header_index = 0; table->header_size = MAX_HEADER_SIZE; table->headers = CALLOC(struct http_decoder_header, table->header_size); + http_decoder_table_init(table); + return table; } @@ -312,6 +341,28 @@ void http_decoder_table_reset(struct http_decoder_table *table, enum http_item t } } +void http_decoder_table_reinit(struct http_decoder_table *table) +{ + if (NULL == table) { + return; + } + + struct http_decoder_header *header = NULL; + assert(table); + + http_decoder_string_reinit(&table->uri); + http_decoder_string_reinit(&table->status); + http_decoder_string_reinit(&table->method); + http_decoder_string_reinit(&table->version); + for (size_t i = 0; i < table->header_iter; i++) { + header = &table->headers[i]; + http_decoder_string_reinit(&header->key); + http_decoder_string_reinit(&header->val); + } + + http_decoder_string_reinit(&table->body); +} + void http_decoder_table_dump(struct http_decoder_table *table) { if (NULL == table) { @@ -380,6 +431,21 @@ int http_decoder_table_get_body(struct http_decoder_table *table, struct hstring return http_decoder_string_get(&table->body, out); } +int http_decoder_table_has_parsed_header(struct http_decoder_table *table) +{ + if (NULL == table) { + return 0; + } + + struct http_decoder_header *tmp_header = &table->headers[table->header_iter]; + if (http_decoder_string_state(&tmp_header->key) == STRING_STATE_COMMIT && + http_decoder_string_state(&tmp_header->val) == STRING_STATE_COMMIT) { + return 1; + } + + return 0; +} + int http_decoder_table_get_header(struct http_decoder_table *table, struct hstring *key, struct http_header *hdr_array, size_t array_size) { @@ -390,12 +456,12 @@ int http_decoder_table_get_header(struct http_decoder_table *table, struct hstri int header_cnt = 0; for (int i = 0; i < table->header_size && header_cnt < array_size; i++) { struct http_decoder_header *tmp_header = &table->headers[i]; - if (NULL == tmp_header) { + if (tmp_header->key.commit.str_len != key->str_len) { continue; } - if (http_decoder_string_state(&tmp_header->key) == STRING_STATE_COMMIT - && http_decoder_string_state(&tmp_header->val) == STRING_STATE_COMMIT) { + if (http_decoder_string_state(&tmp_header->key) == STRING_STATE_COMMIT && + http_decoder_string_state(&tmp_header->val) == STRING_STATE_COMMIT) { struct hstring tmp_key; http_decoder_string_get(&tmp_header->key, &tmp_key); @@ -422,13 +488,15 @@ int http_decoder_table_iter_header(struct http_decoder_table *table, return 0; } - struct http_decoder_header *tmp_header = &table->headers[table->header_iter++]; + struct http_decoder_header *tmp_header = &table->headers[table->header_iter]; if (tmp_header != NULL) { if (http_decoder_string_state(&tmp_header->key) == STRING_STATE_COMMIT && http_decoder_string_state(&tmp_header->val) == STRING_STATE_COMMIT) { http_decoder_string_get(&tmp_header->key, &hdr->key); http_decoder_string_get(&tmp_header->val, &hdr->val); + table->header_iter++; + return 1; } } diff --git a/src/http_decoder/http_decoder_table.h b/src/http_decoder/http_decoder_table.h index 19a4811..a600110 100644 --- a/src/http_decoder/http_decoder_table.h +++ b/src/http_decoder/http_decoder_table.h @@ -21,7 +21,6 @@ extern "C" { #include "http_decoder.h" #include "http_decoder_string.h" - enum http_item { HTTP_ITEM_URI = 0x01, HTTP_ITEM_STATUS = 0x02, @@ -33,7 +32,6 @@ enum http_item { }; struct http_decoder_table; - struct http_decoder_table *http_decoder_table_new(); void http_decoder_table_free(struct http_decoder_table *table); @@ -50,6 +48,8 @@ void http_decoder_table_commit(struct http_decoder_table *table, enum http_item void http_decoder_table_reset(struct http_decoder_table *table, enum http_item type); +void http_decoder_table_reinit(struct http_decoder_table *table); + void http_decoder_table_dump(struct http_decoder_table *table); int http_decoder_table_get_uri(struct http_decoder_table *table, struct hstring *out); @@ -62,10 +62,15 @@ int http_decoder_table_get_version(struct http_decoder_table *table, struct hstr int http_decoder_table_get_body(struct http_decoder_table *table, struct hstring *out); -int http_decoder_table_get_header(struct http_decoder_table *table, struct hstring *key, - struct http_header *hdr_array, size_t array_size); +int http_decoder_table_has_parsed_header(struct http_decoder_table *table); + +int http_decoder_table_get_header(struct http_decoder_table *table, + struct hstring *key, + struct http_header *hdr_array, + size_t array_size); -int http_decoder_table_iter_header(struct http_decoder_table *table, struct http_header *hdr); +int http_decoder_table_iter_header(struct http_decoder_table *table, + struct http_header *hdr); #ifdef __cplusplus } diff --git a/test/http_decoder/CMakeLists.txt b/test/http_decoder/CMakeLists.txt index 9bfc7ee..8135961 100644 --- a/test/http_decoder/CMakeLists.txt +++ b/test/http_decoder/CMakeLists.txt @@ -106,7 +106,10 @@ add_test(NAME HTTP_GET_REQ_PIPELINE_TEST COMMAND ${TEST_MAIN} ${CMAKE_CURRENT_SO add_test(NAME HTTP_TRANS_PIPELINE_TEST COMMAND ${TEST_MAIN} ${CMAKE_CURRENT_SOURCE_DIR}/test_result_json/http_trans_pipeline.json -f "find ${CMAKE_CURRENT_SOURCE_DIR}/http_pcap/ -name http_trans_pipeline.pcap|sort -V" WORKING_DIRECTORY ${TEST_RUN_DIR}) +# add_test(NAME COREDUMP_TEST COMMAND ${TEST_MAIN} ${CMAKE_CURRENT_SOURCE_DIR}/test_result_json/coredump.json +# -f "find ${CMAKE_CURRENT_SOURCE_DIR}/http_pcap/ -name coredump.pcap|sort -V" WORKING_DIRECTORY ${TEST_RUN_DIR}) +# set_tests_properties(COREDUMP_TEST PROPERTIES FIXTURES_REQUIRED TestFixture) set_tests_properties(HTTP_GET_SINGLE_TRANS_TEST HTTP_GET_MULTI_TRANS_TEST HTTP_GET_LONG_COOKIE_TEST diff --git a/test/http_decoder/http_decoder_gtest.cpp b/test/http_decoder/http_decoder_gtest.cpp index 852844e..554f109 100644 --- a/test/http_decoder/http_decoder_gtest.cpp +++ b/test/http_decoder/http_decoder_gtest.cpp @@ -44,32 +44,36 @@ int g_topic_id = 0; void output_http_req_line(struct http_request_line *req_line) { char tmp_str[MAX_KEY_STR_LEN] = {0}; - snprintf(tmp_str, req_line->method.str_len + 1, "%s", req_line->method.str); + memcpy(tmp_str, req_line->method.str, req_line->method.str_len); printf("req_method:%s\n", tmp_str); memset(tmp_str, 0, sizeof(tmp_str)); - snprintf(tmp_str, req_line->uri.str_len + 1, "%s", req_line->uri.str); + memcpy(tmp_str, req_line->uri.str, req_line->uri.str_len); printf("req_uri:%s\n", tmp_str); memset(tmp_str, 0, sizeof(tmp_str)); - snprintf(tmp_str, req_line->version.str_len + 1, "%s", req_line->version.str); + memcpy(tmp_str, req_line->version.str, req_line->version.str_len); printf("req_version:%s\n", tmp_str); } void output_http_res_line(struct http_response_line *res_line) { char tmp_str[MAX_KEY_STR_LEN] = {0}; - snprintf(tmp_str, res_line->version.str_len + 1, "%s", res_line->version.str); + memcpy(tmp_str, res_line->version.str, res_line->version.str_len); printf("res_version:%s\n", tmp_str); memset(tmp_str, 0, sizeof(tmp_str)); - snprintf(tmp_str, res_line->status.str_len + 1, "%s", res_line->status.str); + memcpy(tmp_str, res_line->status.str, res_line->status.str_len); printf("res_status:%s\n", tmp_str); } void output_http_header(struct http_header *header) { - printf("<%s:%s>\n", header->key.str, header->val.str); + char tmp_key[MAX_KEY_STR_LEN] = {0}; + char tmp_val[MAX_KEY_STR_LEN] = {0}; + memcpy(tmp_key, header->key.str, header->key.str_len); + memcpy(tmp_val, header->val.str, header->val.str_len); + printf("<%s:%s>\n", tmp_key, tmp_val); } #endif @@ -165,7 +169,7 @@ http_decoder_test_entry(struct session *sess, int topic_id, const void *data, struct hstring body = {0}; struct http_message *msg = (struct http_message *)data; enum http_message_type msg_type = http_message_type(msg); - + if (msg_type == HTTP_MESSAGE_REQ_LINE || msg_type == HTTP_MESSAGE_REQ_HEADER) { exdata_idx = g_req_exdata_idx; @@ -195,7 +199,6 @@ next: case HTTP_MESSAGE_REQ_HEADER: while (http_message_request_header_next(msg, &header) > 0) { http_header_to_json(json, &header); - // output_http_header(&header); } g_header_count = 1; break; @@ -213,7 +216,6 @@ next: case HTTP_MESSAGE_RES_HEADER: while (http_message_response_header_next(msg, &header) > 0) { http_header_to_json(json, &header); - // output_http_header(&header); } g_header_count = 1; break; diff --git a/test/http_decoder/test_result_json/http_get_long_cookie.json b/test/http_decoder/test_result_json/http_get_long_cookie.json index de07582..535aeef 100644 --- a/test/http_decoder/test_result_json/http_get_long_cookie.json +++ b/test/http_decoder/test_result_json/http_get_long_cookie.json @@ -13,7 +13,11 @@ "Referer": "http://imgcache.qq.com/tencentvideo_v1/player/TPout.swf?max_age=86400&v=20140714", "Accept-Encoding": "gzip,deflate,sdch", "Accept-Language": "zh-CN,zh;q=0.8", - "Cookie": "flashuser=95621BA8CB862E09; piao_city=179; lv_irt_id=3628e1bbe25a6c941da9fac02ec2df8b; cm_cookie=V1,10017&-EP5mRruXhQarsCl5LD-2YzgjVTvyr2K&AQEBh7uoLMUB9lnaB5Tz9XdYnGIWflXmsDrU&150723&150723,10035&7t-tEmfJ076VAsM9&AQEBh7uoLMUB9lnc4tpW7vbazqdrRdBYOUCi&150724&150807,110054&ucO0Z0gctNn3&AQEBh7uoLMUB9llxMNl45F3RAIsKK0iMOJAG&150716&151008,10040&ACZ1r0A70NaEFcGT&AQEBh7uoLMUB9lmVgSoTwuuXZi896zSVsXIF&150818&151014,110015&1&AQEBh7uoLMUB9lkt2LUHO6ARwODHLI_Y51rj&150928&151103,10037&1433388364186289251984&AQEBh7uoLMUB9llIBencOqTAEh2aQ2SURSSQ&150909&151110,10011&jL40Z03uUFI0&AQEBh7uoLMUB9lkfw2sJVNx9g12Fzs12rPSN&150717&151125,10016&F64E6LFZs0W&AQEBh7uoLMUB9llE4yoPFNUykSj7WKaRK5lH&150805&151127,10019&WQAO-C1K9qI5OP8W_t2pSw&AQEBh7uoLMUB9llhpZE87GmOk3XGo_MJgV6K&150826&151130,10015&820490997316506147&AQEBh7uoLMUB9llXiynsGYRhMO3XuPnkuHUt&150715&151201,10012&x3X1yY6b&AQEBh7uoLMUB9ll9mraU_LJCDBYsE0Sbk_V9&151202&151202,110065&ucO0Z0gctNn3&AQEBh7uoLMUB9lkJcK3KDBQTKF0YfZ5wB7r5&150716&151203,110066&jL40Z03uUFI0&AQEBh7uoLMUB9lnyvKSYhcJD1X_rSs_DLVWx&150916&151221,10013&ePyYB2MSKa0TCbebpxKjmU&AQEBh7uoLMUB9ln6_6nGNidqml4nFKXhtE58&151221&151221,110061&d9cfa518d82abee&AQEBh7uoLMUB9llj2NYzmCjxaLWXALTcAGIH&150818&151224,10038&CAESEPZbUhToZJ39CS9MlgXGUSQ&AQEBh7uoLMUB9lmhnrDM5lIGtl6vc1NxMD6F&151110&151224,10077&820490997316506147&AQEBh7uoLMUB9lmkUdUe2xSHGkvM0IRu9Jt9&151214&151228,10008&0yPSvk92ie1nhB8wTUlTq&AQEBh7uoLMUB9lnL5ZCYvXJNvlv53G0CKEkj&150817&151228,10045&0&AQEBh7uoLMUB9llW3v1Vh7W72lv14RlAjUXn&151023&151228,110064&jL40Z03uUFI0&AQEBh7uoLMUB9lkBYuCUDLDrOcGURJcilogv&151016&160104,110069&26d49ecc&AQEBh7uoLMUB9lmlBLTxQY9BkCmimkMFqTo5&151204&160105,10079&B8hGto5y1e3uDXwCMsIun3rjk--dVCof&AQEBh7uoLMUB9llxnFrhDtdNMjZ1hs1il5J4&151214&160105; LHTturn=24; ptisp=ctc; RK=hRWyd82Gd8; pgv_pvi=7567882240; image_md5=bd21d5fb2f401b37cf3a02724dc06545; LTPturn=27; pt2gguin=o0583115900; uin=o0583115900; skey=@Mp9aCinaO; ptcz=10d4b1b7bde835d64663338a8008fd4f81e2c6b5f0ba81a90da3627ee617c7ee; pgv_info=ssid=s4768939310; pgv_pvid=6872592818; o_cookie=583115900; lv_play_index_textAd=47; lv_play_indexl.=32; dc_vplaying=1; LKBturn=29; Lturn=29; adid=583115900; appuser=95621BA8CB862E09; o_minduid=phhdxyNLkxBWMa74VTm5zU4y5EbUv5vR; appuser_95621BA8CB862E09_0=2b7gwp=1453219199_6&2btemv=1455551999_1&2c8311=1453305599_3&2cfx4j=1453651199_3&2cfx9l=1453651199_1&2d49y9=1453823999_2&2d67kl=1454255999_2&2d69mf=1454255999_3&2dxv8l=1455465599_6&2dzhfl=1452614399_1&f_pogvwp=1452095999_1&f_pogvwv=1452095999_2&f_pogw0m=1452095999_1&fd_15bm2t7=1452095999_1&fd_1h2pbsd=1452095999_2&fd_1k6so62=1452095999_1&fd_rhmjmq=1452095999_2&m_roiw0t=1452095999_3&m_xty8wl=1452095999_1&pogree=1452095999_2; TX.boid=100655474=1452072582_1&701041365=1452072585_1; appuser_95621BA8CB862E09_effect_0=fd_1ez2rcc=1452095999_1&fd_qdh7zw=1452095999_1&fd_ul215j=1452095999_1; psessionid=ca7f9c5b_1452071982_583115900_30754; psessiontime=1452071990", "name": "HTTP_DECODER_RESULT_1" + }, + { + "Tuple4": "202.127.156.91.27282>14.17.32.203.80", + "Cookie": "flashuser=95621BA8CB862E09; piao_city=179; lv_irt_id=3628e1bbe25a6c941da9fac02ec2df8b; cm_cookie=V1,10017&-EP5mRruXhQarsCl5LD-2YzgjVTvyr2K&AQEBh7uoLMUB9lnaB5Tz9XdYnGIWflXmsDrU&150723&150723,10035&7t-tEmfJ076VAsM9&AQEBh7uoLMUB9lnc4tpW7vbazqdrRdBYOUCi&150724&150807,110054&ucO0Z0gctNn3&AQEBh7uoLMUB9llxMNl45F3RAIsKK0iMOJAG&150716&151008,10040&ACZ1r0A70NaEFcGT&AQEBh7uoLMUB9lmVgSoTwuuXZi896zSVsXIF&150818&151014,110015&1&AQEBh7uoLMUB9lkt2LUHO6ARwODHLI_Y51rj&150928&151103,10037&1433388364186289251984&AQEBh7uoLMUB9llIBencOqTAEh2aQ2SURSSQ&150909&151110,10011&jL40Z03uUFI0&AQEBh7uoLMUB9lkfw2sJVNx9g12Fzs12rPSN&150717&151125,10016&F64E6LFZs0W&AQEBh7uoLMUB9llE4yoPFNUykSj7WKaRK5lH&150805&151127,10019&WQAO-C1K9qI5OP8W_t2pSw&AQEBh7uoLMUB9llhpZE87GmOk3XGo_MJgV6K&150826&151130,10015&820490997316506147&AQEBh7uoLMUB9llXiynsGYRhMO3XuPnkuHUt&150715&151201,10012&x3X1yY6b&AQEBh7uoLMUB9ll9mraU_LJCDBYsE0Sbk_V9&151202&151202,110065&ucO0Z0gctNn3&AQEBh7uoLMUB9lkJcK3KDBQTKF0YfZ5wB7r5&150716&151203,110066&jL40Z03uUFI0&AQEBh7uoLMUB9lnyvKSYhcJ", + "name": "HTTP_DECODER_RESULT_2" } ]
\ No newline at end of file diff --git a/test/http_decoder/test_result_json/http_headers_exceed_maximum.json b/test/http_decoder/test_result_json/http_headers_exceed_maximum.json index fac220b..70f1b8a 100644 --- a/test/http_decoder/test_result_json/http_headers_exceed_maximum.json +++ b/test/http_decoder/test_result_json/http_headers_exceed_maximum.json @@ -2,7 +2,7 @@ { "Tuple4": "10.0.0.1.61462>10.0.0.2.80", "method": "GET", - "uri": "/x/xx/xxxxxxxxxxxxxxxxxxx/x/xxxxxx/xxxxxxxxxxxxxxx?xxx=1&xxx=1&x=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&vmf=xxxxxxxxxx.xxx.xxx.xxx&ce=UTF-8&ns=xxxxxxxxxx&pageName=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&g=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.jsp&r=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&events=xxxxxxxxxxxxxxxxxxxxxxxxxxx&products=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&v1=xxxxxxxxxxxxxxx&v2=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&v17=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_cxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&c49=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&AQE=1", + "uri": "/x/xx/xxxxxxxxxxxxxxxxxxx/x/xxxxxx/xxxxxxxxxxxxxxx?xxx=1&xxx=1&x=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&vmf=xxxxxxxxxx.xxx.xxx.xxx&ce=UTF-8&ns=xxxxxxxxxx&pageName=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&g=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.jsp&r=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&events=xxxxxxxxxxxxxxxxxxxxxxxxxxx&products=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&v1=xxxxxxxxxxxxxxx&v2=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&v17=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "req_version": "1.1", "major_version": 1, "minor_version": 1, @@ -13,11 +13,15 @@ "Referer": "http://www.xxxxxxxxxx.xxx/xx/xxxxxxxxxxxxxxxxx/xxxxxxxxxxxxxxxxxxxxxx/xxxxxxxxxxxx.jsp", "Accept-Encoding": "gzip,deflate,sdch", "Accept-Language": "en-US,en;q=0.8,en-GB;q=0.6", - "Cookie": "xxxxxxxxxxxxxxxxxxx=ie; xxxxxxxxxxxxxxxxxxxxxx=true; lp=xxxxxx; rememberUn=false; xxx.xxxxxxxxxx.xxxxxxxxxx=xx; xxxxx=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx; autocomplete=1; xxxx=xxxx; xxxx=xxxxv1|xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx; xxxxxx=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", "name": "HTTP_DECODER_RESULT_1" }, { "Tuple4": "10.0.0.1.61462>10.0.0.2.80", + "Cookie": "xxxxxxxxxxxxxxxxxxx=ie; xxxxxxxxxxxxxxxxxxxxxx=true; lp=xxxxxx; rememberUn=false; xxx.xxxxxxxxxx.xxxxxxxxxx=xx; xxxxx=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx; autocomplete=1; xxxx=xxxx; xxxx=xxxxv1|xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx; xxxxxx=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "name": "HTTP_DECODER_RESULT_2" + }, + { + "Tuple4": "10.0.0.1.61462>10.0.0.2.80", "res_version": "1.1", "res_status": "OK", "major_version": 1, @@ -40,6 +44,6 @@ "Keep-Alive": "timeout=15", "Connection": "Keep-Alive", "Content-Type": "image/gif", - "name": "HTTP_DECODER_RESULT_2" + "name": "HTTP_DECODER_RESULT_3" } ]
\ No newline at end of file diff --git a/test/http_decoder/test_result_json/http_req_1byte_sliding_window.json b/test/http_decoder/test_result_json/http_req_1byte_sliding_window.json index 1612574..2d8ae29 100644 --- a/test/http_decoder/test_result_json/http_req_1byte_sliding_window.json +++ b/test/http_decoder/test_result_json/http_req_1byte_sliding_window.json @@ -7,10 +7,22 @@ "major_version": 1, "minor_version": 1, "User-Agent": "Wget/1.14 (linux-gnu)", + "name": "HTTP_DECODER_RESULT_1" + }, + { + "Tuple4": "192.168.40.137.46180>192.168.42.40.80", "Accept": "*/*", + "name": "HTTP_DECODER_RESULT_2" + }, + { + "Tuple4": "192.168.40.137.46180>192.168.42.40.80", "Host": "192.168.42.40", + "name": "HTTP_DECODER_RESULT_3" + }, + { + "Tuple4": "192.168.40.137.46180>192.168.42.40.80", "Connection": "Keep-Alive", - "name": "HTTP_DECODER_RESULT_1" + "name": "HTTP_DECODER_RESULT_4" }, { "Tuple4": "192.168.40.137.46180>192.168.42.40.80", @@ -20,10 +32,26 @@ "minor_version": 0, "status_code": 200, "Server": "SimpleHTTP/0.6 Python/2.7.5", + "name": "HTTP_DECODER_RESULT_5" + }, + { + "Tuple4": "192.168.40.137.46180>192.168.42.40.80", "Date": "Fri, 29 Dec 2023 09:11:12 GMT", + "name": "HTTP_DECODER_RESULT_6" + }, + { + "Tuple4": "192.168.40.137.46180>192.168.42.40.80", "Content-type": "text/html", + "name": "HTTP_DECODER_RESULT_7" + }, + { + "Tuple4": "192.168.40.137.46180>192.168.42.40.80", "Content-Length": "144", + "name": "HTTP_DECODER_RESULT_8" + }, + { + "Tuple4": "192.168.40.137.46180>192.168.42.40.80", "Last-Modified": "Fri, 29 Dec 2023 08:50:53 GMT", - "name": "HTTP_DECODER_RESULT_2" + "name": "HTTP_DECODER_RESULT_9" } ]
\ No newline at end of file diff --git a/test/http_decoder/test_result_json/http_res_1byte_sliding_window.json b/test/http_decoder/test_result_json/http_res_1byte_sliding_window.json index 845e1b2..78285b2 100644 --- a/test/http_decoder/test_result_json/http_res_1byte_sliding_window.json +++ b/test/http_decoder/test_result_json/http_res_1byte_sliding_window.json @@ -20,10 +20,26 @@ "minor_version": 0, "status_code": 200, "Server": "SimpleHTTP/0.6 Python/2.7.5", + "name": "HTTP_DECODER_RESULT_2" + }, + { + "Tuple4": "192.168.42.40.36338>192.168.40.137.80", "Date": "Fri, 29 Dec 2023 09:32:21 GMT", + "name": "HTTP_DECODER_RESULT_3" + }, + { + "Tuple4": "192.168.42.40.36338>192.168.40.137.80", "Content-type": "text/html", + "name": "HTTP_DECODER_RESULT_4" + }, + { + "Tuple4": "192.168.42.40.36338>192.168.40.137.80", "Content-Length": "144", + "name": "HTTP_DECODER_RESULT_5" + }, + { + "Tuple4": "192.168.42.40.36338>192.168.40.137.80", "Last-Modified": "Fri, 29 Dec 2023 08:50:53 GMT", - "name": "HTTP_DECODER_RESULT_2" + "name": "HTTP_DECODER_RESULT_6" } ]
\ No newline at end of file diff --git a/test/http_decoder/test_result_json/http_trans_pipeline.json b/test/http_decoder/test_result_json/http_trans_pipeline.json index e2d18c7..97092c2 100644 --- a/test/http_decoder/test_result_json/http_trans_pipeline.json +++ b/test/http_decoder/test_result_json/http_trans_pipeline.json @@ -85,16 +85,13 @@ }, { "Tuple4": "223.72.39.14.2545>192.168.182.147.80", - "res_version": "1.1", - "res_status": "Not Found", + "method": "GET", + "uri": "/_vti_bin/shtml.dll", + "req_version": "1.1", "major_version": 1, "minor_version": 1, - "status_code": 404, - "Server": "nginx", - "Date": "Thu, 29 Oct 2020 09:59:02 GMT", - "Content-Type": "text/html", - "Content-Length": "146", - "Connection": "keep-alive", + "Host": "116.181.2.152", + "User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)", "name": "HTTP_DECODER_RESULT_8" }, { @@ -169,18 +166,25 @@ }, { "Tuple4": "223.72.39.14.2545>192.168.182.147.80", - "method": "GET", - "uri": "/_vti_bin/shtml.dll", - "req_version": "1.1", + "res_version": "1.1", + "res_status": "Not Found", "major_version": 1, "minor_version": 1, - "Host": "116.181.2.152", - "User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)", + "status_code": 404, + "Server": "nginx", + "Date": "Thu, 29 Oct 2020 09:59:02 GMT", + "Content-Type": "text/html", + "Content-Length": "146", "Connection": "keep-alive", "name": "HTTP_DECODER_RESULT_14" }, { "Tuple4": "223.72.39.14.2545>192.168.182.147.80", + "Connection": "keep-alive", + "name": "HTTP_DECODER_RESULT_15" + }, + { + "Tuple4": "223.72.39.14.2545>192.168.182.147.80", "method": "GET", "uri": "/_vti_bin/shtml.exe", "req_version": "1.1", @@ -189,7 +193,7 @@ "Host": "116.181.2.152", "User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)", "Connection": "keep-alive", - "name": "HTTP_DECODER_RESULT_15" + "name": "HTTP_DECODER_RESULT_16" }, { "Tuple4": "223.72.39.14.2545>192.168.182.147.80", @@ -201,7 +205,7 @@ "Host": "116.181.2.152", "User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)", "Connection": "keep-alive", - "name": "HTTP_DECODER_RESULT_16" + "name": "HTTP_DECODER_RESULT_17" }, { "Tuple4": "223.72.39.14.2545>192.168.182.147.80", @@ -213,7 +217,7 @@ "Host": "116.181.2.152", "User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)", "Connection": "keep-alive", - "name": "HTTP_DECODER_RESULT_17" + "name": "HTTP_DECODER_RESULT_18" }, { "Tuple4": "223.72.39.14.2545>192.168.182.147.80", @@ -225,7 +229,7 @@ "Host": "116.181.2.152", "User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)", "Connection": "keep-alive", - "name": "HTTP_DECODER_RESULT_18" + "name": "HTTP_DECODER_RESULT_19" }, { "Tuple4": "223.72.39.14.2545>192.168.182.147.80", @@ -237,7 +241,7 @@ "Host": "116.181.2.152", "User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)", "Connection": "keep-alive", - "name": "HTTP_DECODER_RESULT_19" + "name": "HTTP_DECODER_RESULT_20" }, { "Tuple4": "223.72.39.14.2545>192.168.182.147.80", @@ -249,7 +253,7 @@ "Host": "116.181.2.152", "User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)", "Connection": "keep-alive", - "name": "HTTP_DECODER_RESULT_20" + "name": "HTTP_DECODER_RESULT_21" }, { "Tuple4": "223.72.39.14.2545>192.168.182.147.80", @@ -261,7 +265,7 @@ "Host": "116.181.2.152", "User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)", "Connection": "keep-alive", - "name": "HTTP_DECODER_RESULT_21" + "name": "HTTP_DECODER_RESULT_22" }, { "Tuple4": "223.72.39.14.2545>192.168.182.147.80", @@ -273,7 +277,7 @@ "Host": "116.181.2.152", "User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)", "Connection": "keep-alive", - "name": "HTTP_DECODER_RESULT_22" + "name": "HTTP_DECODER_RESULT_23" }, { "Tuple4": "223.72.39.14.2545>192.168.182.147.80", @@ -287,7 +291,7 @@ "Content-Type": "text/html", "Content-Length": "146", "Connection": "keep-alive", - "name": "HTTP_DECODER_RESULT_23" + "name": "HTTP_DECODER_RESULT_24" }, { "Tuple4": "223.72.39.14.2545>192.168.182.147.80", @@ -301,7 +305,7 @@ "Content-Type": "text/html", "Content-Length": "146", "Connection": "keep-alive", - "name": "HTTP_DECODER_RESULT_24" + "name": "HTTP_DECODER_RESULT_25" }, { "Tuple4": "223.72.39.14.2545>192.168.182.147.80", @@ -315,7 +319,7 @@ "Content-Type": "text/html", "Content-Length": "146", "Connection": "keep-alive", - "name": "HTTP_DECODER_RESULT_25" + "name": "HTTP_DECODER_RESULT_26" }, { "Tuple4": "223.72.39.14.2545>192.168.182.147.80", @@ -329,7 +333,7 @@ "Content-Type": "text/html", "Content-Length": "146", "Connection": "keep-alive", - "name": "HTTP_DECODER_RESULT_26" + "name": "HTTP_DECODER_RESULT_27" }, { "Tuple4": "223.72.39.14.2545>192.168.182.147.80", @@ -343,6 +347,6 @@ "Content-Type": "text/html", "Content-Length": "146", "Connection": "keep-alive", - "name": "HTTP_DECODER_RESULT_27" + "name": "HTTP_DECODER_RESULT_28" } ]
\ No newline at end of file |
