#2 完善HttpChunk的解析与转发功能

1. 改进HttpResponse接口，增加BodySegment的概念，对应HttpChunk的解析。每个
BodySegment对应一个HttpChunk；
2. 增加HexDump函数，便于调试时输出二进制流的十六进制表示。

6 files changed, 150 insertions, 58 deletions

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 0ba788f..909fb73 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -24,4 +24,6 @@ target_link_libraries(tfe-library maatframe MESA_handle_logger)
 
 add_executable(tfe main.cc)
 target_compile_definitions(tfe PUBLIC -DELPP_THREAD_SAFE -DELPP_FEATURE_ALL)
-target_link_libraries(tfe tfe-library)
\ No newline at end of file
+target_link_libraries(tfe tfe-library)
+
+install(TARGETS tfe RUNTIME DESTINATION ./)
\ No newline at end of file
diff --git a/src/http.h b/src/http.h
index ab57d7c..21bf4ac 100644
--- a/src/http.h
+++ b/src/http.h
@@ -277,11 +277,9 @@ public:
     using body_content_ptr_t = std::unique_ptr<body_content_t>;
 
     /* Body读取、设置接口 */
-    virtual const body_content_t * Body() const = 0;
-    virtual void Body(body_content_ptr_t body) = 0;
-
-    /* Body的Stolen接口 */
-    virtual body_content_ptr_t StolenBody() = 0;
+    virtual const std::vector<const body_content_t *> Body() const = 0;
+    virtual void Body(std::vector<body_content_ptr_t> body) = 0;
+    virtual std::vector<body_content_ptr_t> StolenBody() = 0;
 
     /* ReadOnly，标记本请求为只读。
      * 当一个请求为只读请求时，业务不应修改它的内容，底层处理Readonly的请求时，应直接转发不缓存 */
diff --git a/src/http1.cc b/src/http1.cc
index 2011bd6..6b388ac 100644
--- a/src/http1.cc
+++ b/src/http1.cc
@@ -302,9 +302,7 @@ int Http1RequestParserCallbacks::CallbackOnBody(http_parser * parser, const char
     }
 
     /* 增加length长度的缓冲区，并在尾部追加 */
-    request_ptr->body_content_->reserve(length + request_ptr->body_content_->size());
-    std::copy_n(at, length, std::back_inserter(*request_ptr->body_content_));
-
+    request_ptr->body_content_->insert(request_ptr->body_content_->end(), at, at + length);
     return 0;
 }
 
@@ -323,10 +321,10 @@ int Http1RequestParserCallbacks::CallbackOnHeaderComplete(http_parser * parser)
     /* 拼合URL，TODO: 处理拼合过程中的各种异常，例如多一个斜线、少一个斜线等问题 */
     __request->headers_.ForEachValueOfHeader("Host",
         [__request](const std::string & field, const std::string & value)
-    {
-        __request->str_url_ = value + __request->str_uri_;
-        return false;
-    });
+        {
+            __request->str_url_ = value + __request->str_uri_;
+            return false;
+        });
 
     __request->request_header_complete_ = true;
     return 0;
@@ -520,12 +518,38 @@ public:
     const HttpHeaders & cHeaders() const override
     { return headers_; }
 
-    const body_content_t * Body() const override
-    { return body_content_.get(); }
-    void Body(body_content_ptr_t body) override
-    { body_content_ = std::move(body); }
-    body_content_ptr_t StolenBody() override
-    { return std::move(body_content_); }
+    const std::vector<const body_content_t *> Body() const override
+    {
+        std::vector<const body_content_t *> body_const;
+        for (auto & body_content_iter : body_contents_)
+        {
+            if (!body_content_iter.is_complete) continue;
+            body_const.push_back(body_content_iter.ptr.get());
+        }
+
+        return body_const;
+    }
+
+    std::vector<body_content_ptr_t> StolenBody() override
+    {
+        std::vector<body_content_ptr_t> body_ret;
+        for (auto & body_content_iter : body_contents_)
+        {
+            body_ret.push_back(std::move(body_content_iter.ptr));
+        }
+
+        body_contents_.clear();
+        return body_ret;
+    }
+
+    void Body(std::vector<body_content_ptr_t> body) override
+    {
+        body_contents_.clear();
+        for (auto & body_content_iter : body)
+        {
+            body_contents_.emplace_back(std::move(body_content_iter), true);
+        }
+    }
 
 private:
     int resp_code_;
@@ -544,7 +568,16 @@ private:
     bool construct_body_{true};
 
     /* Body */
-    body_content_ptr_t body_content_{nullptr};
+    struct __body_content
+    {
+        __body_content(body_content_ptr_t __ptr, bool __is_complete)
+            : ptr(std::move(__ptr)), is_complete(__is_complete) {}
+
+        body_content_ptr_t ptr{nullptr};
+        bool is_complete{false};
+    };
+
+    std::vector<struct __body_content> body_contents_;
 
     /* Chunk */
     bool body_is_chunk{false};
@@ -568,6 +601,18 @@ private:
 private:
     void __set_section_state(section_t section, section_state_t state)
     { section_state[section] = state; }
+
+    void __new_body_content(size_t content_length)
+    {
+        body_contents_.emplace_back(std::make_unique<body_content_t>(), false);
+        __last_body_content().ptr->reserve(content_length);
+    }
+
+    struct __body_content & __last_body_content()
+    { return body_contents_.back(); }
+
+    bool __is_body_content_empty()
+    { return body_contents_.empty(); }
 };
 
 class Http1ResponseParserCallbacks
@@ -575,7 +620,6 @@ class Http1ResponseParserCallbacks
 public:
     static const http_parser_settings * CallbackSettings();
     static int CallbackOnMessageBegin(http_parser * parser);
-    static int CallbackOnStatus(http_parser * parser, const char * at, size_t length);
     static int CallbackOnHeaderField(http_parser * parser, const char * at, size_t length);
     static int CallbackOnHeaderValue(http_parser * parser, const char * at, size_t length);
     static int CallbackOnBody(http_parser * parser, const char * at, size_t length);
@@ -592,18 +636,18 @@ private:
 const http_parser_settings * Http1ResponseParserCallbacks::CallbackSettings()
 {
     static struct http_parser_settings __parser_setting =
-    {
-        on_message_begin : CallbackOnMessageBegin, /* on_message_begin */
-        on_url : nullptr, /* on_url */
-        on_status : nullptr, /* on_status */
-        on_header_field : CallbackOnHeaderField, /* on_header_field */
-        on_header_value : CallbackOnHeaderValue, /* on_header_value */
-        on_headers_complete : CallbackOnHeaderComplete, /* on_headers_complete */
-        on_body : CallbackOnBody, /* on_body */
-        on_message_complete : CallbackOnMessageComplete, /* on_message_complete */
-        on_chunk_header : CallbackOnChunkHeader, /* on_chunk_header */
-        on_chunk_complete : CallbackOnChunkComplete
-    };
+        {
+            on_message_begin : CallbackOnMessageBegin, /* on_message_begin */
+            on_url : nullptr, /* on_url */
+            on_status : nullptr, /* on_status */
+            on_header_field : CallbackOnHeaderField, /* on_header_field */
+            on_header_value : CallbackOnHeaderValue, /* on_header_value */
+            on_headers_complete : CallbackOnHeaderComplete, /* on_headers_complete */
+            on_body : CallbackOnBody, /* on_body */
+            on_message_complete : CallbackOnMessageComplete, /* on_message_complete */
+            on_chunk_header : CallbackOnChunkHeader, /* on_chunk_header */
+            on_chunk_complete : CallbackOnChunkComplete
+        };
 
     return &__parser_setting;
 }
@@ -650,13 +694,15 @@ int Http1ResponseParserCallbacks::CallbackOnBody(http_parser * parser, const cha
     resp_ptr->__set_section_state(resp_ptr->kSectionBody, resp_ptr->kStateReading);
 
     /* 第一次回调Body，分配内存，预留内存空间  */
-    if (resp_ptr->body_content_ == nullptr)
+    if (resp_ptr->__is_body_content_empty())
     {
-        resp_ptr->body_content_ = std::make_unique<std::vector<char>>();
-        resp_ptr->body_content_->reserve(parser->content_length);
+        resp_ptr->__new_body_content(parser->content_length);
     }
 
-    std::copy_n(at, length, std::back_inserter(*resp_ptr->body_content_));
+    /* 追加数据 */
+    auto & __last_body_content = resp_ptr->__last_body_content().ptr;
+    __last_body_content->insert(__last_body_content->end(), at, at + length);
+
     return 0;
 }
 
@@ -677,15 +723,24 @@ int Http1ResponseParserCallbacks::CallbackOnChunkHeader(http_parser * parser)
     resp_ptr->body_is_chunk = true;
     resp_ptr->body_chunk_id++;
 
-    /* Drop old body content */
-    resp_ptr->body_content_ = nullptr;
+    /* 增加新的Body分节 */
+    resp_ptr->__new_body_content(parser->content_length);
     return 0;
 }
 
 int Http1ResponseParserCallbacks::CallbackOnChunkComplete(http_parser * parser)
 {
+    /* Body状态设置为Complete，允许上层处理 */
     auto * resp_ptr = __response_this_ptr(parser);
     resp_ptr->__set_section_state(resp_ptr->kSectionBody, resp_ptr->kStateComplete);
+
+    /* 最后一个Body分节的状态置为完成，表示该Body分节数据完成 */
+    auto & __last_body_content = resp_ptr->__last_body_content();
+
+    /* 断言，该分节不应该是完整的，否则出现了多次调用 */
+    assert(!__last_body_content.is_complete);
+    __last_body_content.is_complete = true;
+
     return 0;
 }
 
@@ -728,7 +783,7 @@ ssize_t Http1Response::ConstructFromEvBuf(struct evbuffer * evbuf_ptr)
     evbuffer_remove_buffer(evbuf_ptr, reserved_buffer, static_cast<size_t>(forward_len));
 
     /* 保存在上下文中 */
-    if(evbuf_content_raw_ == nullptr)
+    if (evbuf_content_raw_ == nullptr)
     {
         evbuf_content_raw_ = evbuffer_unique_ptr_t(reserved_buffer);
     }
diff --git a/src/httpscan.cc b/src/httpscan.cc
index 29ee773..777f988 100644
--- a/src/httpscan.cc
+++ b/src/httpscan.cc
@@ -257,23 +257,27 @@ void HttpScanSession::ScanResponseBody(HttpSession * http_session_ctx)
 {
     auto & response = http_session_ctx->response();
 
-    /* Body Content and length, prepare for maat */
-    const char * body_content_raw = response.Body()->data();
-    size_t body_content_length = response.Body()->size();
+    for(const auto & body_segment : response.Body())
+    {
+        const auto * body_content_raw = body_segment->data();
+        const auto body_content_length = body_segment->size();
 
-    auto scan_result = scan_body(body_content_raw, body_content_length,
-        httpscan_module_ref_.table_id_ctrl_http_res_body);
+        auto scan_result = scan_body(body_content_raw, body_content_length,
+            httpscan_module_ref_.table_id_ctrl_http_res_body);
 
-    /* Hit */
-    if (scan_result == scan_result_t::kScanResultHit)
-    {
-        http_session_ctx->SetResponseBodyTag(http_session_ctx->kCallbackTagRepeat);
-        return hit_config_and_do_action(http_session_ctx);
-    }
-    /* Error */
-    else if (scan_result == scan_result_t::kScanResultError)
-    {
-        return hit_scan_error();
+        /* Hit */
+        if (scan_result == scan_result_t::kScanResultHit)
+        {
+            http_session_ctx->SetResponseBodyTag(http_session_ctx->kCallbackTagRepeat);
+            return hit_config_and_do_action(http_session_ctx);
+        }
+            /* Error */
+        else if (scan_result == scan_result_t::kScanResultError)
+        {
+            return hit_scan_error();
+        }
+
+        CLOG(DEBUG, "HttpScanTrace") << hexdump("ContentBody", body_content_raw, body_content_length);
     }
 
     return;
diff --git a/src/main.cc b/src/main.cc
index ea4b6c5..8440962 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -299,6 +299,7 @@ main(int argc, char *argv[])
 
     el::Loggers::getLogger("structLogger");
     el::Loggers::getLogger("conntrace");
+    el::Loggers::getLogger("HttpScanTrace");
 
 	if (nat_getdefaultname())
 	{
@@ -691,7 +692,7 @@ main(int argc, char *argv[])
 	argc -= optind;
 	argv += optind;
 
-    tfe_config_load_from_file(opts, "./config.ini");
+    tfe_config_load_from_file(opts, "conf/tfe.conf");
 
 	/* usage checks before defaults */
 	if (opts->detach && OPTS_DEBUG(opts)) {
@@ -917,7 +918,7 @@ main(int argc, char *argv[])
     g_tfe_instance->http_module = std::make_unique<Http>();
     g_tfe_instance->http_scan_module = std::make_unique<HttpScan>(g_tfe_instance, g_tfe_config);
 
-    g_tfe_instance->stat_module = tfe_stat_create("./config.ini");
+    g_tfe_instance->stat_module = tfe_stat_create("conf/tfe.conf");
     if (g_tfe_instance->stat_module == NULL)
     {
         log_err_printf("Failed to create stat module, exited.");
diff --git a/src/util.h b/src/util.h
index 1e41a90..62e6a82 100644
--- a/src/util.h
+++ b/src/util.h
@@ -212,4 +212,36 @@ struct sockaddr * to_sockaddr_ptr(T * ss, typename std::enable_if<
     std::is_same<T, struct sockaddr_in6>::value>::type * = 0)
 {
     return static_cast<struct sockaddr *>(static_cast<void *>(ss));
-}
\ No newline at end of file
+}
+
+static std::string hexdump(const char * title, const void * buf, unsigned int len)
+{
+    auto * data = static_cast<const unsigned char *>(buf);
+
+    constexpr static size_t LINE_LEN = 80;
+    char line[LINE_LEN];    /* space needed 8+16*3+3+16 == 75 */
+
+    std::string str_ret = string_format("%s at [%p], len=%u\n", (title)? title : "  Dump data", data, len);
+    unsigned int ofs = 0;
+    int i, out;
+
+    while (ofs < len)
+    {
+        /* format the line in the buffer, then use printf to output to screen */
+        out = snprintf(line, LINE_LEN, "%08X:", ofs);
+        for (i = 0; ((ofs + i) < len) && (i < 16); i++)
+            out += snprintf(line+out, LINE_LEN - out, " %02X", (data[ofs+i] & 0xff));
+        for(; i <= 16; i++)
+            out += snprintf(line+out, LINE_LEN - out, " | ");
+        for(i = 0; (ofs < len) && (i < 16); i++, ofs++) {
+            unsigned char c = data[ofs];
+            if ( (c < ' ') || (c > '~'))
+                c = '.';
+            out += snprintf(line+out, LINE_LEN - out, "%c", c);
+        }
+
+        str_ret += string_format("%s\n", line);
+    }
+
+    return str_ret;
+}