summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorliuwentan <[email protected]>2024-01-26 17:21:36 +0800
committerliuwentan <[email protected]>2024-02-01 19:43:09 +0800
commit06af77fb35a55754768f91fbf1363e2dd7906683 (patch)
treefd09d0c99e479bd3a18df4e4ef346702334fbb45
parent889f024485870e988d6185affff31306843b566a (diff)
[HTTP_DECODER]optimize http header cache
-rw-r--r--src/http_decoder/http_content_decompress.c4
-rw-r--r--src/http_decoder/http_decoder.c163
-rw-r--r--src/http_decoder/http_decoder_half.c75
-rw-r--r--src/http_decoder/http_decoder_inc.h26
-rw-r--r--src/http_decoder/http_decoder_string.c120
-rw-r--r--src/http_decoder/http_decoder_string.h6
-rw-r--r--src/http_decoder/http_decoder_table.c80
-rw-r--r--src/http_decoder/http_decoder_table.h15
-rw-r--r--test/http_decoder/CMakeLists.txt3
-rw-r--r--test/http_decoder/http_decoder_gtest.cpp20
-rw-r--r--test/http_decoder/test_result_json/http_get_long_cookie.json6
-rw-r--r--test/http_decoder/test_result_json/http_headers_exceed_maximum.json10
-rw-r--r--test/http_decoder/test_result_json/http_req_1byte_sliding_window.json32
-rw-r--r--test/http_decoder/test_result_json/http_res_1byte_sliding_window.json18
-rw-r--r--test/http_decoder/test_result_json/http_trans_pipeline.json56
15 files changed, 471 insertions, 163 deletions
diff --git a/src/http_decoder/http_content_decompress.c b/src/http_decoder/http_content_decompress.c
index b887487..c12cdf4 100644
--- a/src/http_decoder/http_content_decompress.c
+++ b/src/http_decoder/http_content_decompress.c
@@ -221,8 +221,8 @@ http_content_decompress_write_br(struct http_content_decompress *decompress,
}
}
- if (ret == BROTLI_DECODER_RESULT_SUCCESS
- || ret == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) {
+ if (ret == BROTLI_DECODER_RESULT_SUCCESS ||
+ ret == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) {
return 0;
}
diff --git a/src/http_decoder/http_decoder.c b/src/http_decoder/http_decoder.c
index 523253b..1ce4b86 100644
--- a/src/http_decoder/http_decoder.c
+++ b/src/http_decoder/http_decoder.c
@@ -55,10 +55,31 @@ struct http_decoder_context {
int fs_incoming_bytes_id;
int fs_incoming_pkts_id;
int fs_incoming_trans_id;
+ int fs_err_pkts_id;
struct fieldstat_easy *fse;
struct stellar *st;
};
+struct http_message *
+http_message_new(enum http_message_type type, void *data)
+{
+ struct http_message *msg = CALLOC(struct http_message, 1);
+
+ msg->type = type;
+ msg->data = data;
+
+ return msg;
+}
+
+static void http_message_free(void *http_msg, void *cb_arg)
+{
+ if (NULL == http_msg) {
+ return;
+ }
+
+ FREE(http_msg);
+}
+
static void http_event_handler(enum http_event event,
struct http_decoder_half_data **data,
struct http_event_context *ev_ctx)
@@ -94,23 +115,17 @@ static void http_event_handler(enum http_event event,
*data = half_data;
break;
case HTTP_EVENT_REQ_LINE:
- msg = CALLOC(struct http_message, 1);
- msg->type = HTTP_MESSAGE_REQ_LINE;
- msg->data = *data;
+ msg = http_message_new(HTTP_MESSAGE_REQ_LINE, *data);
session_mq_publish_message(ev_ctx->ref_session, ev_ctx->topic_id, msg);
break;
case HTTP_EVENT_REQ_HDR_END:
- msg = CALLOC(struct http_message, 1);
- msg->type = HTTP_MESSAGE_REQ_HEADER;
- msg->data = *data;
+ msg = http_message_new(HTTP_MESSAGE_REQ_HEADER, *data);
session_mq_publish_message(ev_ctx->ref_session, ev_ctx->topic_id, msg);
break;
case HTTP_EVENT_REQ_BODY_BEGIN:
break;
case HTTP_EVENT_REQ_BODY_DATA:
- msg = CALLOC(struct http_message, 1);
- msg->type = HTTP_MESSAGE_REQ_BODY;
- msg->data = *data;
+ msg = http_message_new(HTTP_MESSAGE_REQ_BODY, *data);
session_mq_publish_message(ev_ctx->ref_session, ev_ctx->topic_id, msg);
break;
case HTTP_EVENT_REQ_BODY_END:
@@ -146,25 +161,19 @@ static void http_event_handler(enum http_event event,
*data = half_data;
break;
case HTTP_EVENT_RES_LINE:
- msg = CALLOC(struct http_message, 1);
- msg->type = HTTP_MESSAGE_RES_LINE;
- msg->data = *data;
+ msg = http_message_new(HTTP_MESSAGE_RES_LINE, *data);
session_mq_publish_message(ev_ctx->ref_session, ev_ctx->topic_id, msg);
break;
case HTTP_EVENT_RES_HDR:
break;
case HTTP_EVENT_RES_HDR_END:
- msg = CALLOC(struct http_message, 1);
- msg->type = HTTP_MESSAGE_RES_HEADER;
- msg->data = *data;
+ msg = http_message_new(HTTP_MESSAGE_RES_HEADER, *data);
session_mq_publish_message(ev_ctx->ref_session, ev_ctx->topic_id, msg);
break;
case HTTP_EVENT_RES_BODY_BEGIN:
break;
case HTTP_EVENT_RES_BODY_DATA:
- msg = CALLOC(struct http_message, 1);
- msg->type = HTTP_MESSAGE_RES_BODY;
- msg->data = *data;
+ msg = http_message_new(HTTP_MESSAGE_RES_BODY, *data);
session_mq_publish_message(ev_ctx->ref_session, ev_ctx->topic_id, msg);
break;
case HTTP_EVENT_RES_BODY_END:
@@ -320,7 +329,9 @@ int http_decoder_entry(struct session *sess, int events,
return -1;
}
+ int thread_id = session_get_current_thread_id(sess);
struct http_decoder_half *cur_half = NULL;
+
if (dir == PACKET_DIRECTION_C2S) {
cur_half = ex_data->decoder->c2s_half;
} else {
@@ -328,38 +339,29 @@ int http_decoder_entry(struct session *sess, int events,
}
http_decoder_half_reinit(cur_half, decoder_ctx->topic_id, ex_data->queue, sess);
- http_decoder_half_parse(cur_half, payload, payload_len);
-
- int thread_id = session_get_current_thread_id(sess);
+ ret = http_decoder_half_parse(cur_half, payload, payload_len);
+ if (ret < 0) {
+ // fieldstat_easy_counter_incrby(decoder_ctx->fse, thread_id,
+ // decoder_ctx->fs_err_pkts_id,
+ // NULL, 0, 1);
+ }
- fieldstat_easy_counter_incrby(decoder_ctx->fse, thread_id,
- decoder_ctx->fs_incoming_bytes_id,
- NULL, 0, payload_len);
+ // fieldstat_easy_counter_incrby(decoder_ctx->fse, thread_id,
+ // decoder_ctx->fs_incoming_bytes_id,
+ // NULL, 0, payload_len);
- fieldstat_easy_counter_incrby(decoder_ctx->fse, thread_id,
- decoder_ctx->fs_incoming_pkts_id,
- NULL, 0, 1);
+ // fieldstat_easy_counter_incrby(decoder_ctx->fse, thread_id,
+ // decoder_ctx->fs_incoming_pkts_id,
+ // NULL, 0, 1);
- long long trans_cnt = http_decoder_half_trans_count(cur_half);
- fieldstat_easy_counter_incrby(decoder_ctx->fse, thread_id,
- decoder_ctx->fs_incoming_trans_id,
- NULL, 0, trans_cnt);
+ // long long trans_cnt = http_decoder_half_trans_count(cur_half);
+ // fieldstat_easy_counter_incrby(decoder_ctx->fse, thread_id,
+ // decoder_ctx->fs_incoming_trans_id,
+ // NULL, 0, trans_cnt);
return 0;
}
-static void http_message_free(void *msg, void *cb_arg)
-{
- if (NULL == msg) {
- return;
- }
-
- struct http_message *message = (struct http_message *)msg;
- message->data = NULL; //don't have memory's ownership
-
- FREE(message);
-}
-
static void _http_decoder_context_free(struct http_decoder_context *ctx)
{
if (NULL == ctx) {
@@ -391,65 +393,84 @@ static void http_decoder_ex_data_free(struct session *s, int idx,
}
#define FS_OUTPUT_INTERVAL_S 1
-void *http_decoder_init(struct stellar *st)
+static int http_decoder_fs_init(struct http_decoder_context *ctx, int thread_num)
{
- struct http_decoder_context *ctx = CALLOC(struct http_decoder_context, 1);
-
- ctx->st = st;
- ctx->ex_data_idx = stellar_session_get_ex_new_index(st, "HTTP_DECODER",
- http_decoder_ex_data_free,
- NULL);
-
- int plugin_id = stellar_plugin_register(st, SESS_EV_TCP|SESS_EV_CLOSING,
- http_decoder_entry, ctx);
- if (plugin_id >= 0) {
- ctx->plugin_id = plugin_id;
- }
-
- int topic_id = session_mq_get_topic_id(st, http_decoder_topic);
- if (topic_id < 0) {
- topic_id = session_mq_create_topic(st, http_decoder_topic,
- http_message_free, NULL);
- }
- ctx->topic_id = topic_id;
-
- int thread_num = stellar_get_worker_thread_num(st);
-
ctx->fse = fieldstat_easy_new(thread_num, "http_decoder_statistics", NULL, 0);
if (NULL == ctx->fse) {
fprintf(stderr, "fieldstat_easy_new failed.");
- goto failed;
+ return -1;
}
ctx->fs_incoming_bytes_id =
fieldstat_easy_register_counter(ctx->fse, "incoming_bytes");
if (ctx->fs_incoming_bytes_id < 0) {
fprintf(stderr, "fieldstat_easy_register_counter incoming_bytes failed.");
- goto failed;
+ return -1;
}
ctx->fs_incoming_trans_id =
fieldstat_easy_register_counter(ctx->fse, "incoming_trans");
if (ctx->fs_incoming_trans_id < 0) {
fprintf(stderr, "fieldstat_easy_register_counter incoming_trans failed.");
- goto failed;
+ return -1;
}
ctx->fs_incoming_pkts_id =
fieldstat_easy_register_counter(ctx->fse, "incoming_pkts");
if (ctx->fs_incoming_pkts_id < 0) {
fprintf(stderr, "fieldstat_easy_register_counter incoming_pkts failed.");
- goto failed;
+ return -1;
+ }
+
+ ctx->fs_err_pkts_id = fieldstat_easy_register_counter(ctx->fse, "err_pkts");
+ if (ctx->fs_err_pkts_id < 0) {
+ fprintf(stderr, "fieldstat_easy_register_counter err_pkts failed.");
+ return -1;
}
int ret = fieldstat_easy_enable_auto_output(ctx->fse, fs_file_name,
FS_OUTPUT_INTERVAL_S);
if (ret < 0) {
fprintf(stderr, "fieldstat_easy_enable_auto_output failed.");
- goto failed;
+ return -1;
}
+
sleep(1);
+ return 0;
+}
+
+void *http_decoder_init(struct stellar *st)
+{
+ struct http_decoder_context *ctx = CALLOC(struct http_decoder_context, 1);
+
+ ctx->st = st;
+ ctx->ex_data_idx = stellar_session_get_ex_new_index(st, "HTTP_DECODER",
+ http_decoder_ex_data_free,
+ NULL);
+
+ int plugin_id = stellar_plugin_register(st, SESS_EV_TCP|SESS_EV_CLOSING,
+ http_decoder_entry, ctx);
+ if (plugin_id < 0) {
+ return NULL;
+ }
+
+ ctx->plugin_id = plugin_id;
+
+ int topic_id = session_mq_get_topic_id(st, http_decoder_topic);
+ if (topic_id < 0) {
+ topic_id = session_mq_create_topic(st, http_decoder_topic,
+ http_message_free, NULL);
+ }
+
+ ctx->topic_id = topic_id;
+
+ int thread_num = stellar_get_worker_thread_num(st);
+
+ if (http_decoder_fs_init(ctx, thread_num) < 0) {
+ goto failed;
+ }
+
printf("http_decoder_init: ex_data_idx:%d, plugin_id:%d, topic_id:%d\n",
ctx->ex_data_idx, ctx->plugin_id, ctx->topic_id);
diff --git a/src/http_decoder/http_decoder_half.c b/src/http_decoder/http_decoder_half.c
index d97cd25..cc13d8e 100644
--- a/src/http_decoder/http_decoder_half.c
+++ b/src/http_decoder/http_decoder_half.c
@@ -10,9 +10,13 @@
#include <assert.h>
#include <stdio.h>
+#include <string.h>
#include "stellar/utils.h"
+#include "stellar/session_mq.h"
#include "llhttp.h"
+#include "http_decoder.h"
+#include "http_decoder_inc.h"
#include "http_decoder_utils.h"
#include "http_decoder_half.h"
#include "http_decoder_table.h"
@@ -43,6 +47,7 @@ struct http_decoder_half {
http_event_cb *http_ev_cb;
struct http_event_context *http_ev_ctx;
long long trans_counter;
+ long long err_counter;
};
// #define HTTP_DECODER_DEBUG
@@ -194,6 +199,7 @@ static int on_method_complete(llhttp_t *http)
STRING_STATE_REFER) {
http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_METHOD);
}
+
http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_METHOD);
return 0;
@@ -226,6 +232,7 @@ static int on_uri_complete(llhttp_t *http)
STRING_STATE_REFER) {
http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_URI);
}
+
http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_URI);
return 0;
@@ -258,6 +265,7 @@ static int on_version_complete(llhttp_t *http)
STRING_STATE_REFER) {
http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_VERSION);
}
+
http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_VERSION);
half->ref_data->major_version = llhttp_get_http_major(&half->parser);
@@ -299,14 +307,16 @@ static int on_status_complete(llhttp_t *http)
if (http_decoder_table_state(half->ref_data->table, HTTP_ITEM_STATUS) ==
STRING_STATE_REFER) {
http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_STATUS);
- }
- http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_STATUS);
+ }
+ http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_STATUS);
half->ref_data->status_code = llhttp_get_status_code(&half->parser);
- half->event = HTTP_EVENT_RES_LINE;
- if (half->http_ev_cb != NULL) {
- half->http_ev_cb(half->event, &half->ref_data, half->http_ev_ctx);
+ if (half->parser.type == HTTP_RESPONSE) {
+ half->event = HTTP_EVENT_RES_LINE;
+ if (half->http_ev_cb != NULL) {
+ half->http_ev_cb(half->event, &half->ref_data, half->http_ev_ctx);
+ }
}
return 0;
@@ -335,10 +345,6 @@ static int on_header_field_complete(llhttp_t *http)
container_of(http, struct http_decoder_half, parser);
assert(half);
- if (http_decoder_table_state(half->ref_data->table, HTTP_ITEM_HDRKEY) ==
- STRING_STATE_REFER) {
- http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_HDRKEY);
- }
http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_HDRKEY);
return 0;
@@ -367,10 +373,11 @@ static int on_header_value_complete(llhttp_t *http)
container_of(http, struct http_decoder_half, parser);
assert(half);
- if (http_decoder_table_state(half->ref_data->table, HTTP_ITEM_HDRVAL) ==
- STRING_STATE_REFER) {
- http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_HDRVAL);
+ if (http_decoder_table_state(half->ref_data->table, HTTP_ITEM_HDRKEY) ==
+ STRING_STATE_CACHE) {
+ http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_HDRKEY);
}
+
http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_HDRVAL);
if (half->ref_data->content_encoding == HTTP_CONTENT_ENCODING_NONE) {
@@ -379,10 +386,9 @@ static int on_header_value_complete(llhttp_t *http)
if (http_decoder_table_get_header(half->ref_data->table, &key,
&http_hdr, 1) == 1) {
- char *str = safe_dup(http_hdr.val.str, http_hdr.val.str_len);
- half->ref_data->content_encoding =
- http_content_encoding_str2int(str);
- FREE(str);
+ char encoding_str[16] = {0};
+ memcpy(encoding_str, http_hdr.val.str, http_hdr.val.str_len);
+ half->ref_data->content_encoding = http_content_encoding_str2int(encoding_str);
}
}
@@ -476,8 +482,8 @@ static int on_body(llhttp_t *http, const char *at, size_t length)
http_decoder_table_reset(half->ref_data->table, HTTP_ITEM_BODY);
}
- http_decoder_table_refer(half->ref_data->table, HTTP_ITEM_BODY, at,
- length);
+ http_decoder_table_refer(half->ref_data->table, HTTP_ITEM_BODY,
+ at, length);
http_decoder_table_commit(half->ref_data->table, HTTP_ITEM_BODY);
}
@@ -510,7 +516,7 @@ static void http_decoder_half_init(struct http_decoder_half *half,
llhttp_settings_init(&half->settings);
llhttp_init(&half->parser, type, &half->settings);
-
+
half->settings.on_message_begin = on_message_begin;
half->settings.on_message_complete = on_message_complete;
half->settings.on_reset = on_reset;
@@ -578,11 +584,35 @@ void http_decoder_half_reinit(struct http_decoder_half *half, int topic_id,
{
assert(half != NULL);
+ if (half->ref_data != NULL) {
+ http_decoder_table_reinit(half->ref_data->table);
+ }
+
half->http_ev_ctx->topic_id = topic_id;
half->http_ev_ctx->ref_session = sess;
half->http_ev_ctx->ref_queue = queue;
}
+static void publish_message_for_parsed_header(struct http_decoder_half *half)
+{
+ if (http_decoder_table_has_parsed_header(half->ref_data->table) <= 0) {
+ return;
+ }
+
+ // publish complete kv-header message
+ struct http_message *msg = NULL;
+ if (half->parser.type == HTTP_REQUEST) {
+ msg = http_message_new(HTTP_MESSAGE_REQ_HEADER, half->ref_data);
+ session_mq_publish_message(half->http_ev_ctx->ref_session,
+ half->http_ev_ctx->topic_id, msg);
+ } else {
+ // http response
+ msg = http_message_new(HTTP_MESSAGE_RES_HEADER, half->ref_data);
+ session_mq_publish_message(half->http_ev_ctx->ref_session,
+ half->http_ev_ctx->topic_id, msg);
+ }
+}
+
int http_decoder_half_parse(struct http_decoder_half *half,
const char *data, size_t data_len)
{
@@ -643,11 +673,18 @@ int http_decoder_half_parse(struct http_decoder_half *half,
if (http_decoder_table_state(half->ref_data->table, HTTP_ITEM_HDRKEY)
== STRING_STATE_REFER) {
+ publish_message_for_parsed_header(half);
http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_HDRKEY);
}
if (http_decoder_table_state(half->ref_data->table, HTTP_ITEM_HDRVAL)
== STRING_STATE_REFER) {
+ publish_message_for_parsed_header(half);
+
+ /* Header key should have been committed
+ If it's not cached, cache it for next packet to use
+ */
+ http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_HDRKEY);
http_decoder_table_cache(half->ref_data->table, HTTP_ITEM_HDRVAL);
}
diff --git a/src/http_decoder/http_decoder_inc.h b/src/http_decoder/http_decoder_inc.h
new file mode 100644
index 0000000..fd39d6e
--- /dev/null
+++ b/src/http_decoder/http_decoder_inc.h
@@ -0,0 +1,26 @@
+/*
+**********************************************************************************************
+* File: http_decoder_inc.h
+* Description:
+* Authors: Liu WenTan <[email protected]>
+* Date: 2024-01-10
+* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
+***********************************************************************************************
+*/
+
+#ifndef _HTTP_DECODER_INC_H_
+#define _HTTP_DECODER_INC_H_
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+struct http_message;
+struct http_message *http_message_new(enum http_message_type type, void *data);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif \ No newline at end of file
diff --git a/src/http_decoder/http_decoder_string.c b/src/http_decoder/http_decoder_string.c
index cfaeb4c..5414c5b 100644
--- a/src/http_decoder/http_decoder_string.c
+++ b/src/http_decoder/http_decoder_string.c
@@ -59,32 +59,83 @@ void http_decoder_string_refer(struct http_decoder_string *rstr,
rstr->state = STRING_STATE_REFER;
}
-void http_decoder_string_cache(struct http_decoder_string *rstr)
+static void string_refer2cache(struct http_decoder_string *rstr)
{
- if (NULL == rstr) {
+ if (0 == rstr->refer.str_len) {
return;
}
- size_t length = 0;
+ if (rstr->cache.str_len >= rstr->max_cache_size) {
+ return;
+ }
- switch (rstr->state) {
- case STRING_STATE_REFER:
- if (rstr->refer.str_len > 0) {
- if (NULL == rstr->cache.str) {
- length = rstr->refer.str_len + 1;
- rstr->cache.str = CALLOC(char, length);
- } else {
- length = rstr->cache.str_len + rstr->refer.str_len + 2;
- rstr->cache.str = REALLOC(char, rstr->cache.str, length);
- }
+ size_t length = rstr->cache.str_len + rstr->refer.str_len;
+ if (length > rstr->max_cache_size) {
+ length = rstr->max_cache_size;
+ }
+
+ if (NULL == rstr->cache.str) {
+ rstr->cache.str = CALLOC(char, length + 1);
+ memcpy(rstr->cache.str, rstr->refer.str, length);
+ } else {
+ rstr->cache.str = REALLOC(char, rstr->cache.str, length + 1);
+ memcpy(rstr->cache.str + rstr->cache.str_len, rstr->refer.str,
+ (length - rstr->cache.str_len));
+ }
+
+ rstr->cache.str_len = length;
+
+ rstr->refer.str = NULL;
+ rstr->refer.str_len = 0;
+}
+
+static void string_commit2cache(struct http_decoder_string *rstr)
+{
+ if (rstr->cache.str_len == rstr->commit.str_len &&
+ rstr->cache.str == rstr->commit.str) {
- memcpy(rstr->cache.str + rstr->cache.str_len, rstr->refer.str,
- rstr->refer.str_len);
- rstr->cache.str_len += rstr->refer.str_len;
+ rstr->commit.str = NULL;
+ rstr->commit.str_len = 0;
+ return;
+ }
- rstr->refer.str = NULL;
- rstr->refer.str_len = 0;
+ //Only http header key need to backward to cache
+ size_t length = 0;
+ if (rstr->commit.str_len > rstr->max_cache_size) {
+ length = rstr->max_cache_size;
+ } else {
+ length = rstr->commit.str_len;
+ }
+
+ if (length > 0) {
+ if (NULL == rstr->cache.str) {
+ rstr->cache.str = CALLOC(char, length + 1);
+ } else {
+ abort();
}
+ memcpy(rstr->cache.str, rstr->commit.str, length);
+ rstr->cache.str_len = length;
+
+ rstr->commit.str = NULL;
+ rstr->commit.str_len = 0;
+ }
+}
+
+void http_decoder_string_cache(struct http_decoder_string *rstr)
+{
+ if (NULL == rstr) {
+ return;
+ }
+
+ switch (rstr->state) {
+ case STRING_STATE_REFER:
+ string_refer2cache(rstr);
+ break;
+ case STRING_STATE_CACHE:
+ break;
+ case STRING_STATE_COMMIT:
+ //commit backward to cache
+ string_commit2cache(rstr);
break;
default:
abort();
@@ -149,6 +200,39 @@ void http_decoder_string_reset(struct http_decoder_string *rstr)
rstr->state = STRING_STATE_INIT;
}
+
+void http_decoder_string_init(struct http_decoder_string *rstr,
+ size_t max_cache_size)
+{
+ rstr->max_cache_size = max_cache_size;
+}
+
+void http_decoder_string_reinit(struct http_decoder_string *rstr)
+{
+ if (rstr->state == STRING_STATE_CACHE) {
+ return;
+ }
+
+ if (rstr->state == STRING_STATE_COMMIT &&
+ rstr->cache.str == rstr->commit.str &&
+ rstr->cache.str_len == rstr->commit.str_len) {
+ return;
+ }
+
+ if (rstr->cache.str != NULL) {
+ FREE(rstr->cache.str);
+ rstr->cache.str_len = 0;
+ }
+
+ rstr->refer.str = NULL;
+ rstr->refer.str_len = 0;
+
+ rstr->commit.str = NULL;
+ rstr->commit.str_len = 0;
+
+ rstr->state = STRING_STATE_INIT;
+}
+
enum string_state http_decoder_string_state(struct http_decoder_string *rstr)
{
return rstr->state;
diff --git a/src/http_decoder/http_decoder_string.h b/src/http_decoder/http_decoder_string.h
index 804a7ff..f9d81dd 100644
--- a/src/http_decoder/http_decoder_string.h
+++ b/src/http_decoder/http_decoder_string.h
@@ -65,6 +65,7 @@ struct http_decoder_string {
struct hstring commit;
enum string_state state;
+ size_t max_cache_size;
};
void http_decoder_string_refer(struct http_decoder_string *rstr,
@@ -76,6 +77,11 @@ void http_decoder_string_commit(struct http_decoder_string *rstr);
void http_decoder_string_reset(struct http_decoder_string *rstr);
+void http_decoder_string_init(struct http_decoder_string *rstr,
+ size_t max_cache_size);
+
+void http_decoder_string_reinit(struct http_decoder_string *rstr);
+
enum string_state http_decoder_string_state(struct http_decoder_string *rstr);
int http_decoder_string_get(struct http_decoder_string *rstr, struct hstring *out);
diff --git a/src/http_decoder/http_decoder_table.c b/src/http_decoder/http_decoder_table.c
index 300994d..ac12296 100644
--- a/src/http_decoder/http_decoder_table.c
+++ b/src/http_decoder/http_decoder_table.c
@@ -19,7 +19,12 @@
#include "stellar/utils.h"
#define MAX_HEADER_SIZE 16
-
+#define MAX_URI_CACHE_SIZE 1024
+#define MAX_STATUS_CACHE_SIZE 32
+#define MAX_METHOD_CACHE_SIZE 8
+#define MAX_VERSION_CACHE_SIZE 4
+#define MAX_HEADER_KEY_CACHE_SIZE 128
+#define MAX_HEADER_VALUE_CACHE_SIZE 1024
struct http_decoder_header {
struct http_decoder_string key;
@@ -39,15 +44,39 @@ struct http_decoder_table {
struct http_decoder_header *headers;
};
+static void http_decoder_table_init(struct http_decoder_table *table)
+{
+ if (NULL == table) {
+ return;
+ }
+
+ struct http_decoder_header *header = NULL;
+ assert(table);
+
+ http_decoder_string_init(&table->uri, MAX_URI_CACHE_SIZE);
+ http_decoder_string_init(&table->status, MAX_STATUS_CACHE_SIZE);
+ http_decoder_string_init(&table->method, MAX_METHOD_CACHE_SIZE);
+ http_decoder_string_init(&table->version, MAX_METHOD_CACHE_SIZE);
+
+ for (size_t i = 0; i < table->header_size; i++) {
+ header = &table->headers[i];
+ http_decoder_string_init(&header->key, MAX_HEADER_KEY_CACHE_SIZE);
+ http_decoder_string_init(&header->val, MAX_HEADER_VALUE_CACHE_SIZE);
+ }
+
+ http_decoder_string_init(&table->body, 0);
+}
+
struct http_decoder_table *http_decoder_table_new()
{
struct http_decoder_table *table = CALLOC(struct http_decoder_table, 1);
assert(table);
- table->header_index = 0;
table->header_size = MAX_HEADER_SIZE;
table->headers = CALLOC(struct http_decoder_header, table->header_size);
+ http_decoder_table_init(table);
+
return table;
}
@@ -312,6 +341,28 @@ void http_decoder_table_reset(struct http_decoder_table *table, enum http_item t
}
}
+void http_decoder_table_reinit(struct http_decoder_table *table)
+{
+ if (NULL == table) {
+ return;
+ }
+
+ struct http_decoder_header *header = NULL;
+ assert(table);
+
+ http_decoder_string_reinit(&table->uri);
+ http_decoder_string_reinit(&table->status);
+ http_decoder_string_reinit(&table->method);
+ http_decoder_string_reinit(&table->version);
+ for (size_t i = 0; i < table->header_iter; i++) {
+ header = &table->headers[i];
+ http_decoder_string_reinit(&header->key);
+ http_decoder_string_reinit(&header->val);
+ }
+
+ http_decoder_string_reinit(&table->body);
+}
+
void http_decoder_table_dump(struct http_decoder_table *table)
{
if (NULL == table) {
@@ -380,6 +431,21 @@ int http_decoder_table_get_body(struct http_decoder_table *table, struct hstring
return http_decoder_string_get(&table->body, out);
}
+int http_decoder_table_has_parsed_header(struct http_decoder_table *table)
+{
+ if (NULL == table) {
+ return 0;
+ }
+
+ struct http_decoder_header *tmp_header = &table->headers[table->header_iter];
+ if (http_decoder_string_state(&tmp_header->key) == STRING_STATE_COMMIT &&
+ http_decoder_string_state(&tmp_header->val) == STRING_STATE_COMMIT) {
+ return 1;
+ }
+
+ return 0;
+}
+
int http_decoder_table_get_header(struct http_decoder_table *table, struct hstring *key,
struct http_header *hdr_array, size_t array_size)
{
@@ -390,12 +456,12 @@ int http_decoder_table_get_header(struct http_decoder_table *table, struct hstri
int header_cnt = 0;
for (int i = 0; i < table->header_size && header_cnt < array_size; i++) {
struct http_decoder_header *tmp_header = &table->headers[i];
- if (NULL == tmp_header) {
+ if (tmp_header->key.commit.str_len != key->str_len) {
continue;
}
- if (http_decoder_string_state(&tmp_header->key) == STRING_STATE_COMMIT
- && http_decoder_string_state(&tmp_header->val) == STRING_STATE_COMMIT) {
+ if (http_decoder_string_state(&tmp_header->key) == STRING_STATE_COMMIT &&
+ http_decoder_string_state(&tmp_header->val) == STRING_STATE_COMMIT) {
struct hstring tmp_key;
http_decoder_string_get(&tmp_header->key, &tmp_key);
@@ -422,13 +488,15 @@ int http_decoder_table_iter_header(struct http_decoder_table *table,
return 0;
}
- struct http_decoder_header *tmp_header = &table->headers[table->header_iter++];
+ struct http_decoder_header *tmp_header = &table->headers[table->header_iter];
if (tmp_header != NULL) {
if (http_decoder_string_state(&tmp_header->key) == STRING_STATE_COMMIT &&
http_decoder_string_state(&tmp_header->val) == STRING_STATE_COMMIT) {
http_decoder_string_get(&tmp_header->key, &hdr->key);
http_decoder_string_get(&tmp_header->val, &hdr->val);
+ table->header_iter++;
+
return 1;
}
}
diff --git a/src/http_decoder/http_decoder_table.h b/src/http_decoder/http_decoder_table.h
index 19a4811..a600110 100644
--- a/src/http_decoder/http_decoder_table.h
+++ b/src/http_decoder/http_decoder_table.h
@@ -21,7 +21,6 @@ extern "C" {
#include "http_decoder.h"
#include "http_decoder_string.h"
-
enum http_item {
HTTP_ITEM_URI = 0x01,
HTTP_ITEM_STATUS = 0x02,
@@ -33,7 +32,6 @@ enum http_item {
};
struct http_decoder_table;
-
struct http_decoder_table *http_decoder_table_new();
void http_decoder_table_free(struct http_decoder_table *table);
@@ -50,6 +48,8 @@ void http_decoder_table_commit(struct http_decoder_table *table, enum http_item
void http_decoder_table_reset(struct http_decoder_table *table, enum http_item type);
+void http_decoder_table_reinit(struct http_decoder_table *table);
+
void http_decoder_table_dump(struct http_decoder_table *table);
int http_decoder_table_get_uri(struct http_decoder_table *table, struct hstring *out);
@@ -62,10 +62,15 @@ int http_decoder_table_get_version(struct http_decoder_table *table, struct hstr
int http_decoder_table_get_body(struct http_decoder_table *table, struct hstring *out);
-int http_decoder_table_get_header(struct http_decoder_table *table, struct hstring *key,
- struct http_header *hdr_array, size_t array_size);
+int http_decoder_table_has_parsed_header(struct http_decoder_table *table);
+
+int http_decoder_table_get_header(struct http_decoder_table *table,
+ struct hstring *key,
+ struct http_header *hdr_array,
+ size_t array_size);
-int http_decoder_table_iter_header(struct http_decoder_table *table, struct http_header *hdr);
+int http_decoder_table_iter_header(struct http_decoder_table *table,
+ struct http_header *hdr);
#ifdef __cplusplus
}
diff --git a/test/http_decoder/CMakeLists.txt b/test/http_decoder/CMakeLists.txt
index 9bfc7ee..8135961 100644
--- a/test/http_decoder/CMakeLists.txt
+++ b/test/http_decoder/CMakeLists.txt
@@ -106,7 +106,10 @@ add_test(NAME HTTP_GET_REQ_PIPELINE_TEST COMMAND ${TEST_MAIN} ${CMAKE_CURRENT_SO
add_test(NAME HTTP_TRANS_PIPELINE_TEST COMMAND ${TEST_MAIN} ${CMAKE_CURRENT_SOURCE_DIR}/test_result_json/http_trans_pipeline.json
-f "find ${CMAKE_CURRENT_SOURCE_DIR}/http_pcap/ -name http_trans_pipeline.pcap|sort -V" WORKING_DIRECTORY ${TEST_RUN_DIR})
+# add_test(NAME COREDUMP_TEST COMMAND ${TEST_MAIN} ${CMAKE_CURRENT_SOURCE_DIR}/test_result_json/coredump.json
+# -f "find ${CMAKE_CURRENT_SOURCE_DIR}/http_pcap/ -name coredump.pcap|sort -V" WORKING_DIRECTORY ${TEST_RUN_DIR})
+# set_tests_properties(COREDUMP_TEST PROPERTIES FIXTURES_REQUIRED TestFixture)
set_tests_properties(HTTP_GET_SINGLE_TRANS_TEST
HTTP_GET_MULTI_TRANS_TEST
HTTP_GET_LONG_COOKIE_TEST
diff --git a/test/http_decoder/http_decoder_gtest.cpp b/test/http_decoder/http_decoder_gtest.cpp
index 852844e..554f109 100644
--- a/test/http_decoder/http_decoder_gtest.cpp
+++ b/test/http_decoder/http_decoder_gtest.cpp
@@ -44,32 +44,36 @@ int g_topic_id = 0;
void output_http_req_line(struct http_request_line *req_line)
{
char tmp_str[MAX_KEY_STR_LEN] = {0};
- snprintf(tmp_str, req_line->method.str_len + 1, "%s", req_line->method.str);
+ memcpy(tmp_str, req_line->method.str, req_line->method.str_len);
printf("req_method:%s\n", tmp_str);
memset(tmp_str, 0, sizeof(tmp_str));
- snprintf(tmp_str, req_line->uri.str_len + 1, "%s", req_line->uri.str);
+ memcpy(tmp_str, req_line->uri.str, req_line->uri.str_len);
printf("req_uri:%s\n", tmp_str);
memset(tmp_str, 0, sizeof(tmp_str));
- snprintf(tmp_str, req_line->version.str_len + 1, "%s", req_line->version.str);
+ memcpy(tmp_str, req_line->version.str, req_line->version.str_len);
printf("req_version:%s\n", tmp_str);
}
void output_http_res_line(struct http_response_line *res_line)
{
char tmp_str[MAX_KEY_STR_LEN] = {0};
- snprintf(tmp_str, res_line->version.str_len + 1, "%s", res_line->version.str);
+ memcpy(tmp_str, res_line->version.str, res_line->version.str_len);
printf("res_version:%s\n", tmp_str);
memset(tmp_str, 0, sizeof(tmp_str));
- snprintf(tmp_str, res_line->status.str_len + 1, "%s", res_line->status.str);
+ memcpy(tmp_str, res_line->status.str, res_line->status.str_len);
printf("res_status:%s\n", tmp_str);
}
void output_http_header(struct http_header *header)
{
- printf("<%s:%s>\n", header->key.str, header->val.str);
+ char tmp_key[MAX_KEY_STR_LEN] = {0};
+ char tmp_val[MAX_KEY_STR_LEN] = {0};
+ memcpy(tmp_key, header->key.str, header->key.str_len);
+ memcpy(tmp_val, header->val.str, header->val.str_len);
+ printf("<%s:%s>\n", tmp_key, tmp_val);
}
#endif
@@ -165,7 +169,7 @@ http_decoder_test_entry(struct session *sess, int topic_id, const void *data,
struct hstring body = {0};
struct http_message *msg = (struct http_message *)data;
enum http_message_type msg_type = http_message_type(msg);
-
+
if (msg_type == HTTP_MESSAGE_REQ_LINE ||
msg_type == HTTP_MESSAGE_REQ_HEADER) {
exdata_idx = g_req_exdata_idx;
@@ -195,7 +199,6 @@ next:
case HTTP_MESSAGE_REQ_HEADER:
while (http_message_request_header_next(msg, &header) > 0) {
http_header_to_json(json, &header);
- // output_http_header(&header);
}
g_header_count = 1;
break;
@@ -213,7 +216,6 @@ next:
case HTTP_MESSAGE_RES_HEADER:
while (http_message_response_header_next(msg, &header) > 0) {
http_header_to_json(json, &header);
- // output_http_header(&header);
}
g_header_count = 1;
break;
diff --git a/test/http_decoder/test_result_json/http_get_long_cookie.json b/test/http_decoder/test_result_json/http_get_long_cookie.json
index de07582..535aeef 100644
--- a/test/http_decoder/test_result_json/http_get_long_cookie.json
+++ b/test/http_decoder/test_result_json/http_get_long_cookie.json
@@ -13,7 +13,11 @@
"Referer": "http://imgcache.qq.com/tencentvideo_v1/player/TPout.swf?max_age=86400&v=20140714",
"Accept-Encoding": "gzip,deflate,sdch",
"Accept-Language": "zh-CN,zh;q=0.8",
- "Cookie": "flashuser=95621BA8CB862E09; piao_city=179; lv_irt_id=3628e1bbe25a6c941da9fac02ec2df8b; cm_cookie=V1,10017&-EP5mRruXhQarsCl5LD-2YzgjVTvyr2K&AQEBh7uoLMUB9lnaB5Tz9XdYnGIWflXmsDrU&150723&150723,10035&7t-tEmfJ076VAsM9&AQEBh7uoLMUB9lnc4tpW7vbazqdrRdBYOUCi&150724&150807,110054&ucO0Z0gctNn3&AQEBh7uoLMUB9llxMNl45F3RAIsKK0iMOJAG&150716&151008,10040&ACZ1r0A70NaEFcGT&AQEBh7uoLMUB9lmVgSoTwuuXZi896zSVsXIF&150818&151014,110015&1&AQEBh7uoLMUB9lkt2LUHO6ARwODHLI_Y51rj&150928&151103,10037&1433388364186289251984&AQEBh7uoLMUB9llIBencOqTAEh2aQ2SURSSQ&150909&151110,10011&jL40Z03uUFI0&AQEBh7uoLMUB9lkfw2sJVNx9g12Fzs12rPSN&150717&151125,10016&F64E6LFZs0W&AQEBh7uoLMUB9llE4yoPFNUykSj7WKaRK5lH&150805&151127,10019&WQAO-C1K9qI5OP8W_t2pSw&AQEBh7uoLMUB9llhpZE87GmOk3XGo_MJgV6K&150826&151130,10015&820490997316506147&AQEBh7uoLMUB9llXiynsGYRhMO3XuPnkuHUt&150715&151201,10012&x3X1yY6b&AQEBh7uoLMUB9ll9mraU_LJCDBYsE0Sbk_V9&151202&151202,110065&ucO0Z0gctNn3&AQEBh7uoLMUB9lkJcK3KDBQTKF0YfZ5wB7r5&150716&151203,110066&jL40Z03uUFI0&AQEBh7uoLMUB9lnyvKSYhcJD1X_rSs_DLVWx&150916&151221,10013&ePyYB2MSKa0TCbebpxKjmU&AQEBh7uoLMUB9ln6_6nGNidqml4nFKXhtE58&151221&151221,110061&d9cfa518d82abee&AQEBh7uoLMUB9llj2NYzmCjxaLWXALTcAGIH&150818&151224,10038&CAESEPZbUhToZJ39CS9MlgXGUSQ&AQEBh7uoLMUB9lmhnrDM5lIGtl6vc1NxMD6F&151110&151224,10077&820490997316506147&AQEBh7uoLMUB9lmkUdUe2xSHGkvM0IRu9Jt9&151214&151228,10008&0yPSvk92ie1nhB8wTUlTq&AQEBh7uoLMUB9lnL5ZCYvXJNvlv53G0CKEkj&150817&151228,10045&0&AQEBh7uoLMUB9llW3v1Vh7W72lv14RlAjUXn&151023&151228,110064&jL40Z03uUFI0&AQEBh7uoLMUB9lkBYuCUDLDrOcGURJcilogv&151016&160104,110069&26d49ecc&AQEBh7uoLMUB9lmlBLTxQY9BkCmimkMFqTo5&151204&160105,10079&B8hGto5y1e3uDXwCMsIun3rjk--dVCof&AQEBh7uoLMUB9llxnFrhDtdNMjZ1hs1il5J4&151214&160105; LHTturn=24; ptisp=ctc; RK=hRWyd82Gd8; pgv_pvi=7567882240; image_md5=bd21d5fb2f401b37cf3a02724dc06545; LTPturn=27; pt2gguin=o0583115900; uin=o0583115900; skey=@Mp9aCinaO; ptcz=10d4b1b7bde835d64663338a8008fd4f81e2c6b5f0ba81a90da3627ee617c7ee; pgv_info=ssid=s4768939310; pgv_pvid=6872592818; o_cookie=583115900; lv_play_index_textAd=47; lv_play_indexl.=32; dc_vplaying=1; LKBturn=29; Lturn=29; adid=583115900; appuser=95621BA8CB862E09; o_minduid=phhdxyNLkxBWMa74VTm5zU4y5EbUv5vR; appuser_95621BA8CB862E09_0=2b7gwp=1453219199_6&2btemv=1455551999_1&2c8311=1453305599_3&2cfx4j=1453651199_3&2cfx9l=1453651199_1&2d49y9=1453823999_2&2d67kl=1454255999_2&2d69mf=1454255999_3&2dxv8l=1455465599_6&2dzhfl=1452614399_1&f_pogvwp=1452095999_1&f_pogvwv=1452095999_2&f_pogw0m=1452095999_1&fd_15bm2t7=1452095999_1&fd_1h2pbsd=1452095999_2&fd_1k6so62=1452095999_1&fd_rhmjmq=1452095999_2&m_roiw0t=1452095999_3&m_xty8wl=1452095999_1&pogree=1452095999_2; TX.boid=100655474=1452072582_1&701041365=1452072585_1; appuser_95621BA8CB862E09_effect_0=fd_1ez2rcc=1452095999_1&fd_qdh7zw=1452095999_1&fd_ul215j=1452095999_1; psessionid=ca7f9c5b_1452071982_583115900_30754; psessiontime=1452071990",
"name": "HTTP_DECODER_RESULT_1"
+ },
+ {
+ "Tuple4": "202.127.156.91.27282>14.17.32.203.80",
+ "Cookie": "flashuser=95621BA8CB862E09; piao_city=179; lv_irt_id=3628e1bbe25a6c941da9fac02ec2df8b; cm_cookie=V1,10017&-EP5mRruXhQarsCl5LD-2YzgjVTvyr2K&AQEBh7uoLMUB9lnaB5Tz9XdYnGIWflXmsDrU&150723&150723,10035&7t-tEmfJ076VAsM9&AQEBh7uoLMUB9lnc4tpW7vbazqdrRdBYOUCi&150724&150807,110054&ucO0Z0gctNn3&AQEBh7uoLMUB9llxMNl45F3RAIsKK0iMOJAG&150716&151008,10040&ACZ1r0A70NaEFcGT&AQEBh7uoLMUB9lmVgSoTwuuXZi896zSVsXIF&150818&151014,110015&1&AQEBh7uoLMUB9lkt2LUHO6ARwODHLI_Y51rj&150928&151103,10037&1433388364186289251984&AQEBh7uoLMUB9llIBencOqTAEh2aQ2SURSSQ&150909&151110,10011&jL40Z03uUFI0&AQEBh7uoLMUB9lkfw2sJVNx9g12Fzs12rPSN&150717&151125,10016&F64E6LFZs0W&AQEBh7uoLMUB9llE4yoPFNUykSj7WKaRK5lH&150805&151127,10019&WQAO-C1K9qI5OP8W_t2pSw&AQEBh7uoLMUB9llhpZE87GmOk3XGo_MJgV6K&150826&151130,10015&820490997316506147&AQEBh7uoLMUB9llXiynsGYRhMO3XuPnkuHUt&150715&151201,10012&x3X1yY6b&AQEBh7uoLMUB9ll9mraU_LJCDBYsE0Sbk_V9&151202&151202,110065&ucO0Z0gctNn3&AQEBh7uoLMUB9lkJcK3KDBQTKF0YfZ5wB7r5&150716&151203,110066&jL40Z03uUFI0&AQEBh7uoLMUB9lnyvKSYhcJ",
+ "name": "HTTP_DECODER_RESULT_2"
}
] \ No newline at end of file
diff --git a/test/http_decoder/test_result_json/http_headers_exceed_maximum.json b/test/http_decoder/test_result_json/http_headers_exceed_maximum.json
index fac220b..70f1b8a 100644
--- a/test/http_decoder/test_result_json/http_headers_exceed_maximum.json
+++ b/test/http_decoder/test_result_json/http_headers_exceed_maximum.json
@@ -2,7 +2,7 @@
{
"Tuple4": "10.0.0.1.61462>10.0.0.2.80",
"method": "GET",
- "uri": "/x/xx/xxxxxxxxxxxxxxxxxxx/x/xxxxxx/xxxxxxxxxxxxxxx?xxx=1&xxx=1&x=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&vmf=xxxxxxxxxx.xxx.xxx.xxx&ce=UTF-8&ns=xxxxxxxxxx&pageName=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&g=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.jsp&r=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&events=xxxxxxxxxxxxxxxxxxxxxxxxxxx&products=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&v1=xxxxxxxxxxxxxxx&v2=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&v17=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx_cxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&c49=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&AQE=1",
+ "uri": "/x/xx/xxxxxxxxxxxxxxxxxxx/x/xxxxxx/xxxxxxxxxxxxxxx?xxx=1&xxx=1&x=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&vmf=xxxxxxxxxx.xxx.xxx.xxx&ce=UTF-8&ns=xxxxxxxxxx&pageName=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&g=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.jsp&r=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&events=xxxxxxxxxxxxxxxxxxxxxxxxxxx&products=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&v1=xxxxxxxxxxxxxxx&v2=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&v17=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
"req_version": "1.1",
"major_version": 1,
"minor_version": 1,
@@ -13,11 +13,15 @@
"Referer": "http://www.xxxxxxxxxx.xxx/xx/xxxxxxxxxxxxxxxxx/xxxxxxxxxxxxxxxxxxxxxx/xxxxxxxxxxxx.jsp",
"Accept-Encoding": "gzip,deflate,sdch",
"Accept-Language": "en-US,en;q=0.8,en-GB;q=0.6",
- "Cookie": "xxxxxxxxxxxxxxxxxxx=ie; xxxxxxxxxxxxxxxxxxxxxx=true; lp=xxxxxx; rememberUn=false; xxx.xxxxxxxxxx.xxxxxxxxxx=xx; xxxxx=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx; autocomplete=1; xxxx=xxxx; xxxx=xxxxv1|xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx; xxxxxx=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
"name": "HTTP_DECODER_RESULT_1"
},
{
"Tuple4": "10.0.0.1.61462>10.0.0.2.80",
+ "Cookie": "xxxxxxxxxxxxxxxxxxx=ie; xxxxxxxxxxxxxxxxxxxxxx=true; lp=xxxxxx; rememberUn=false; xxx.xxxxxxxxxx.xxxxxxxxxx=xx; xxxxx=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx; autocomplete=1; xxxx=xxxx; xxxx=xxxxv1|xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx; xxxxxx=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx2xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+ "name": "HTTP_DECODER_RESULT_2"
+ },
+ {
+ "Tuple4": "10.0.0.1.61462>10.0.0.2.80",
"res_version": "1.1",
"res_status": "OK",
"major_version": 1,
@@ -40,6 +44,6 @@
"Keep-Alive": "timeout=15",
"Connection": "Keep-Alive",
"Content-Type": "image/gif",
- "name": "HTTP_DECODER_RESULT_2"
+ "name": "HTTP_DECODER_RESULT_3"
}
] \ No newline at end of file
diff --git a/test/http_decoder/test_result_json/http_req_1byte_sliding_window.json b/test/http_decoder/test_result_json/http_req_1byte_sliding_window.json
index 1612574..2d8ae29 100644
--- a/test/http_decoder/test_result_json/http_req_1byte_sliding_window.json
+++ b/test/http_decoder/test_result_json/http_req_1byte_sliding_window.json
@@ -7,10 +7,22 @@
"major_version": 1,
"minor_version": 1,
"User-Agent": "Wget/1.14 (linux-gnu)",
+ "name": "HTTP_DECODER_RESULT_1"
+ },
+ {
+ "Tuple4": "192.168.40.137.46180>192.168.42.40.80",
"Accept": "*/*",
+ "name": "HTTP_DECODER_RESULT_2"
+ },
+ {
+ "Tuple4": "192.168.40.137.46180>192.168.42.40.80",
"Host": "192.168.42.40",
+ "name": "HTTP_DECODER_RESULT_3"
+ },
+ {
+ "Tuple4": "192.168.40.137.46180>192.168.42.40.80",
"Connection": "Keep-Alive",
- "name": "HTTP_DECODER_RESULT_1"
+ "name": "HTTP_DECODER_RESULT_4"
},
{
"Tuple4": "192.168.40.137.46180>192.168.42.40.80",
@@ -20,10 +32,26 @@
"minor_version": 0,
"status_code": 200,
"Server": "SimpleHTTP/0.6 Python/2.7.5",
+ "name": "HTTP_DECODER_RESULT_5"
+ },
+ {
+ "Tuple4": "192.168.40.137.46180>192.168.42.40.80",
"Date": "Fri, 29 Dec 2023 09:11:12 GMT",
+ "name": "HTTP_DECODER_RESULT_6"
+ },
+ {
+ "Tuple4": "192.168.40.137.46180>192.168.42.40.80",
"Content-type": "text/html",
+ "name": "HTTP_DECODER_RESULT_7"
+ },
+ {
+ "Tuple4": "192.168.40.137.46180>192.168.42.40.80",
"Content-Length": "144",
+ "name": "HTTP_DECODER_RESULT_8"
+ },
+ {
+ "Tuple4": "192.168.40.137.46180>192.168.42.40.80",
"Last-Modified": "Fri, 29 Dec 2023 08:50:53 GMT",
- "name": "HTTP_DECODER_RESULT_2"
+ "name": "HTTP_DECODER_RESULT_9"
}
] \ No newline at end of file
diff --git a/test/http_decoder/test_result_json/http_res_1byte_sliding_window.json b/test/http_decoder/test_result_json/http_res_1byte_sliding_window.json
index 845e1b2..78285b2 100644
--- a/test/http_decoder/test_result_json/http_res_1byte_sliding_window.json
+++ b/test/http_decoder/test_result_json/http_res_1byte_sliding_window.json
@@ -20,10 +20,26 @@
"minor_version": 0,
"status_code": 200,
"Server": "SimpleHTTP/0.6 Python/2.7.5",
+ "name": "HTTP_DECODER_RESULT_2"
+ },
+ {
+ "Tuple4": "192.168.42.40.36338>192.168.40.137.80",
"Date": "Fri, 29 Dec 2023 09:32:21 GMT",
+ "name": "HTTP_DECODER_RESULT_3"
+ },
+ {
+ "Tuple4": "192.168.42.40.36338>192.168.40.137.80",
"Content-type": "text/html",
+ "name": "HTTP_DECODER_RESULT_4"
+ },
+ {
+ "Tuple4": "192.168.42.40.36338>192.168.40.137.80",
"Content-Length": "144",
+ "name": "HTTP_DECODER_RESULT_5"
+ },
+ {
+ "Tuple4": "192.168.42.40.36338>192.168.40.137.80",
"Last-Modified": "Fri, 29 Dec 2023 08:50:53 GMT",
- "name": "HTTP_DECODER_RESULT_2"
+ "name": "HTTP_DECODER_RESULT_6"
}
] \ No newline at end of file
diff --git a/test/http_decoder/test_result_json/http_trans_pipeline.json b/test/http_decoder/test_result_json/http_trans_pipeline.json
index e2d18c7..97092c2 100644
--- a/test/http_decoder/test_result_json/http_trans_pipeline.json
+++ b/test/http_decoder/test_result_json/http_trans_pipeline.json
@@ -85,16 +85,13 @@
},
{
"Tuple4": "223.72.39.14.2545>192.168.182.147.80",
- "res_version": "1.1",
- "res_status": "Not Found",
+ "method": "GET",
+ "uri": "/_vti_bin/shtml.dll",
+ "req_version": "1.1",
"major_version": 1,
"minor_version": 1,
- "status_code": 404,
- "Server": "nginx",
- "Date": "Thu, 29 Oct 2020 09:59:02 GMT",
- "Content-Type": "text/html",
- "Content-Length": "146",
- "Connection": "keep-alive",
+ "Host": "116.181.2.152",
+ "User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)",
"name": "HTTP_DECODER_RESULT_8"
},
{
@@ -169,18 +166,25 @@
},
{
"Tuple4": "223.72.39.14.2545>192.168.182.147.80",
- "method": "GET",
- "uri": "/_vti_bin/shtml.dll",
- "req_version": "1.1",
+ "res_version": "1.1",
+ "res_status": "Not Found",
"major_version": 1,
"minor_version": 1,
- "Host": "116.181.2.152",
- "User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)",
+ "status_code": 404,
+ "Server": "nginx",
+ "Date": "Thu, 29 Oct 2020 09:59:02 GMT",
+ "Content-Type": "text/html",
+ "Content-Length": "146",
"Connection": "keep-alive",
"name": "HTTP_DECODER_RESULT_14"
},
{
"Tuple4": "223.72.39.14.2545>192.168.182.147.80",
+ "Connection": "keep-alive",
+ "name": "HTTP_DECODER_RESULT_15"
+ },
+ {
+ "Tuple4": "223.72.39.14.2545>192.168.182.147.80",
"method": "GET",
"uri": "/_vti_bin/shtml.exe",
"req_version": "1.1",
@@ -189,7 +193,7 @@
"Host": "116.181.2.152",
"User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)",
"Connection": "keep-alive",
- "name": "HTTP_DECODER_RESULT_15"
+ "name": "HTTP_DECODER_RESULT_16"
},
{
"Tuple4": "223.72.39.14.2545>192.168.182.147.80",
@@ -201,7 +205,7 @@
"Host": "116.181.2.152",
"User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)",
"Connection": "keep-alive",
- "name": "HTTP_DECODER_RESULT_16"
+ "name": "HTTP_DECODER_RESULT_17"
},
{
"Tuple4": "223.72.39.14.2545>192.168.182.147.80",
@@ -213,7 +217,7 @@
"Host": "116.181.2.152",
"User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)",
"Connection": "keep-alive",
- "name": "HTTP_DECODER_RESULT_17"
+ "name": "HTTP_DECODER_RESULT_18"
},
{
"Tuple4": "223.72.39.14.2545>192.168.182.147.80",
@@ -225,7 +229,7 @@
"Host": "116.181.2.152",
"User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)",
"Connection": "keep-alive",
- "name": "HTTP_DECODER_RESULT_18"
+ "name": "HTTP_DECODER_RESULT_19"
},
{
"Tuple4": "223.72.39.14.2545>192.168.182.147.80",
@@ -237,7 +241,7 @@
"Host": "116.181.2.152",
"User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)",
"Connection": "keep-alive",
- "name": "HTTP_DECODER_RESULT_19"
+ "name": "HTTP_DECODER_RESULT_20"
},
{
"Tuple4": "223.72.39.14.2545>192.168.182.147.80",
@@ -249,7 +253,7 @@
"Host": "116.181.2.152",
"User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)",
"Connection": "keep-alive",
- "name": "HTTP_DECODER_RESULT_20"
+ "name": "HTTP_DECODER_RESULT_21"
},
{
"Tuple4": "223.72.39.14.2545>192.168.182.147.80",
@@ -261,7 +265,7 @@
"Host": "116.181.2.152",
"User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)",
"Connection": "keep-alive",
- "name": "HTTP_DECODER_RESULT_21"
+ "name": "HTTP_DECODER_RESULT_22"
},
{
"Tuple4": "223.72.39.14.2545>192.168.182.147.80",
@@ -273,7 +277,7 @@
"Host": "116.181.2.152",
"User-Agent": "Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)",
"Connection": "keep-alive",
- "name": "HTTP_DECODER_RESULT_22"
+ "name": "HTTP_DECODER_RESULT_23"
},
{
"Tuple4": "223.72.39.14.2545>192.168.182.147.80",
@@ -287,7 +291,7 @@
"Content-Type": "text/html",
"Content-Length": "146",
"Connection": "keep-alive",
- "name": "HTTP_DECODER_RESULT_23"
+ "name": "HTTP_DECODER_RESULT_24"
},
{
"Tuple4": "223.72.39.14.2545>192.168.182.147.80",
@@ -301,7 +305,7 @@
"Content-Type": "text/html",
"Content-Length": "146",
"Connection": "keep-alive",
- "name": "HTTP_DECODER_RESULT_24"
+ "name": "HTTP_DECODER_RESULT_25"
},
{
"Tuple4": "223.72.39.14.2545>192.168.182.147.80",
@@ -315,7 +319,7 @@
"Content-Type": "text/html",
"Content-Length": "146",
"Connection": "keep-alive",
- "name": "HTTP_DECODER_RESULT_25"
+ "name": "HTTP_DECODER_RESULT_26"
},
{
"Tuple4": "223.72.39.14.2545>192.168.182.147.80",
@@ -329,7 +333,7 @@
"Content-Type": "text/html",
"Content-Length": "146",
"Connection": "keep-alive",
- "name": "HTTP_DECODER_RESULT_26"
+ "name": "HTTP_DECODER_RESULT_27"
},
{
"Tuple4": "223.72.39.14.2545>192.168.182.147.80",
@@ -343,6 +347,6 @@
"Content-Type": "text/html",
"Content-Length": "146",
"Connection": "keep-alive",
- "name": "HTTP_DECODER_RESULT_27"
+ "name": "HTTP_DECODER_RESULT_28"
}
] \ No newline at end of file