Add BODY_START message; add http_url_decode(); disable CONNECT tunnel.HEADv2.0.4dev-on-stellar2.0

10 files changed, 220 insertions, 58 deletions

diff --git a/src/http_decoder.cpp b/src/http_decoder.cpp
index cc7671b..6be2550 100644
--- a/src/http_decoder.cpp
+++ b/src/http_decoder.cpp
@@ -77,18 +77,20 @@ static void http_event_handler(enum http_event event, struct http_decoder_half_d
         *data = half_data;
         queue_idx = http_decoder_result_queue_req_index(queue); //get the index after inc
         /* llhttp always call on_message_begin() even if llhttp_execute() error!!! */
-        msg = http_message_new(HTTP_TRANSACTION_NEW, queue, queue_idx, HTTP_REQUEST);
+        msg = http_message_new(HTTP_TRANSACTION_START, queue, queue_idx, HTTP_REQUEST);
         session_mq_publish_message(ev_ctx->ref_session, exdata->pub_topic_id, msg);      
         http_decoder_stat_update(&httpd_env->hd_stat, thread_id, HTTPD_STAT_TRANSACTION_NEW, 1);  
         break;
     case HTTP_EVENT_REQ_LINE:
         msg = http_message_new(HTTP_MESSAGE_REQ_LINE, queue, queue_idx, HTTP_REQUEST);
         session_mq_publish_message(ev_ctx->ref_session, exdata->pub_topic_id, msg);
-        if(httpd_tunnel_identify(PACKET_DIRECTION_C2S, half_data)){
+        if(httpd_tunnel_identify(httpd_env, PACKET_DIRECTION_C2S, half_data)){
             exdata->tunnel_state = HTTP_TUN_C2S_HDR_START;
             http_decoder_stat_update(&httpd_env->hd_stat, thread_id, HTTPD_STAT_TUNNEL, 1); 
         }
-        http_decoder_get_url(half_data, mempool);
+        if(httpd_is_tunnel_session(exdata)){
+            http_decoder_get_url(half_data, mempool);
+        }
         break;
     case HTTP_EVENT_REQ_HDR:
         msg = http_message_new(HTTP_MESSAGE_REQ_HEADER, queue, queue_idx, HTTP_REQUEST);
@@ -116,7 +118,8 @@ static void http_event_handler(enum http_event event, struct http_decoder_half_d
     }
     break;
     case HTTP_EVENT_REQ_BODY_BEGIN:
-        //do nothing, in BODY_DATA event
+        msg = http_message_new(HTTP_MESSAGE_REQ_BODY_START, queue, queue_idx, HTTP_REQUEST);
+        session_mq_publish_message(ev_ctx->ref_session, exdata->pub_topic_id, msg);
         break;
     case HTTP_EVENT_REQ_BODY_DATA:
         msg = http_message_new(HTTP_MESSAGE_REQ_BODY, queue, queue_idx, HTTP_REQUEST);
@@ -131,7 +134,7 @@ static void http_event_handler(enum http_event event, struct http_decoder_half_d
         session_is_symmetric(ev_ctx->ref_session, &flow_flag);
 
         if(SESSION_SEEN_C2S_FLOW == flow_flag){
-            msg = http_message_new(HTTP_TRANSACTION_FREE, queue, queue_idx, HTTP_REQUEST);
+            msg = http_message_new(HTTP_TRANSACTION_END, queue, queue_idx, HTTP_REQUEST);
             session_mq_publish_message(ev_ctx->ref_session, exdata->pub_topic_id, msg);
             http_decoder_stat_update(&httpd_env->hd_stat, thread_id, HTTPD_STAT_TRANSACTION_FREE, 1);
             http_decoder_stat_update(&httpd_env->hd_stat, thread_id, HTTPD_STAT_ASYMMETRY_TRANSACTION_C2S, 1);
@@ -182,7 +185,7 @@ static void http_event_handler(enum http_event event, struct http_decoder_half_d
         *data = half_data;
         if(0 == session_is_symmetric(ev_ctx->ref_session, &flow_flag)){
             if(SESSION_SEEN_S2C_FLOW == flow_flag){
-                msg = http_message_new(HTTP_TRANSACTION_NEW, queue, queue_idx, HTTP_RESPONSE);
+                msg = http_message_new(HTTP_TRANSACTION_START, queue, queue_idx, HTTP_RESPONSE);
                 session_mq_publish_message(ev_ctx->ref_session, exdata->pub_topic_id, msg);    
             }
         }
@@ -190,7 +193,7 @@ static void http_event_handler(enum http_event event, struct http_decoder_half_d
     case HTTP_EVENT_RES_LINE:
         msg = http_message_new(HTTP_MESSAGE_RES_LINE, queue, queue_idx, HTTP_RESPONSE);
         session_mq_publish_message(ev_ctx->ref_session, exdata->pub_topic_id, msg);
-        if(httpd_tunnel_identify(PACKET_DIRECTION_S2C, half_data)){
+        if(httpd_tunnel_identify(httpd_env,PACKET_DIRECTION_S2C, half_data)){
             exdata->tunnel_state = HTTP_TUN_S2C_START;
         }else{
             //connect response fail, reset tunnel_state
@@ -224,6 +227,8 @@ static void http_event_handler(enum http_event event, struct http_decoder_half_d
     }
         break;
     case HTTP_EVENT_RES_BODY_BEGIN:
+        msg = http_message_new(HTTP_MESSAGE_RES_BODY_START, queue, queue_idx, HTTP_RESPONSE);
+        session_mq_publish_message(ev_ctx->ref_session, exdata->pub_topic_id, msg);    
         break;
     case HTTP_EVENT_RES_BODY_DATA:
         msg = http_message_new(HTTP_MESSAGE_RES_BODY, queue, queue_idx, HTTP_RESPONSE);
@@ -234,7 +239,7 @@ static void http_event_handler(enum http_event event, struct http_decoder_half_d
         session_mq_publish_message(ev_ctx->ref_session, exdata->pub_topic_id, msg);    
         break;
     case HTTP_EVENT_RES_END:
-        msg = http_message_new(HTTP_TRANSACTION_FREE, queue, queue_idx, HTTP_RESPONSE);
+        msg = http_message_new(HTTP_TRANSACTION_END, queue, queue_idx, HTTP_RESPONSE);
         session_mq_publish_message(ev_ctx->ref_session, exdata->pub_topic_id, msg);
         http_decoder_stat_update(&httpd_env->hd_stat, thread_id, HTTPD_STAT_TRANSACTION_FREE, 1); 
         session_is_symmetric(ev_ctx->ref_session, &flow_flag);
@@ -347,11 +352,6 @@ static void _http_decoder_context_free(struct http_decoder_env *env)
     {
         return;
     }
-    if (env->hd_stat.fse != NULL)
-    {
-        fieldstat_easy_free(env->hd_stat.fse);
-        env->hd_stat.fse = NULL;
-    }
 
     http_decoder_stat_free(&env->hd_stat);
 
@@ -436,6 +436,16 @@ static int load_http_decoder_config(const char *cfg_path,
         hd_cfg->stat_output_interval = DEFAULT_STAT_OUTPUT_INTERVAL;
     }
 
+    int_val = toml_int_in(basic_sec_tbl, "proxy_enable");
+    if (int_val.ok != 0)
+    {
+        hd_cfg->proxy_enable = int_val.u.i;
+    }
+    else
+    {
+        hd_cfg->proxy_enable = 0;
+    }
+
     toml_free(root);
     return ret;
 }
@@ -606,7 +616,7 @@ extern "C"
         }
 
         http_decoder_half_reinit(cur_half, exdata->queue, exdata->mempool, sess);
-        int ret = http_decoder_half_parse(cur_half, payload, payload_len);
+        int ret = http_decoder_half_parse(httpd_env->hd_cfg.proxy_enable, cur_half, payload, payload_len);
         if (ret < 0)
         {
             http_decoder_stat_update(&httpd_env->hd_stat, thread_id, HTTPD_STAT_PARSE_ERR, 1);
@@ -617,6 +627,10 @@ extern "C"
     void http_decoder_tunnel_msg_cb(struct session *sess, int topic_id, const void *tmsg, void *per_session_ctx, void *plugin_env)
     {
         struct http_decoder_env *httpd_env = (struct http_decoder_env *)plugin_env;
+        if(0 == httpd_env->hd_cfg.proxy_enable){
+            return;
+        }
+
         struct http_decoder_exdata *exdata = (struct http_decoder_exdata *)session_exdata_get(sess, httpd_env->topic_exdata_compose[HTTPD_TOPIC_HTTP_TUNNEL_INDEX].exdata_id);
         enum http_tunnel_message_type tmsg_type = http_tunnel_message_type_get((const struct http_tunnel_message *)tmsg);
         switch (tmsg_type)
@@ -796,14 +810,6 @@ static const struct http_topic_exdata_compose g_topic_exdata_compose[HTTPD_TOPIC
             goto failed;
         }
         
-        // printf("http decoder init succ, plugin id:%d \n", httpd_env->plugin_id);
-        // for(int i = 0; i < HTTPD_TOPIC_INDEX_MAX; i++){
-        //     printf("\ttopic_name:%s, topic_id:%d, ex_data_name:%s, exdata_id:%d\n", 
-        //             httpd_env->topic_exdata_compose[i].topic_name, 
-        //             httpd_env->topic_exdata_compose[i].sub_topic_id, 
-        //             httpd_env->topic_exdata_compose[i].exdata_name,
-        //             httpd_env->topic_exdata_compose[i].exdata_id);
-        // }
         return httpd_env;
 
     failed:
@@ -831,7 +837,7 @@ static const struct http_topic_exdata_compose g_topic_exdata_compose[HTTPD_TOPIC
         return msg->type;
     }
 
-    void http_message_get_request_line(const struct http_message *msg,
+    void http_message_request_line_get0(const struct http_message *msg,
                                       struct http_request_line *line)
     {
         if (unlikely(NULL == msg || msg->type != HTTP_MESSAGE_REQ_LINE))
@@ -852,7 +858,7 @@ static const struct http_topic_exdata_compose g_topic_exdata_compose[HTTPD_TOPIC
         http_decoder_half_data_get_request_line(req_data, line);
     }
 
-    void http_message_get_response_line(const struct http_message *msg,
+    void http_message_response_line_get0(const struct http_message *msg,
                                        struct http_response_line *line)
     {
         if (unlikely(NULL == msg || msg->type != HTTP_MESSAGE_RES_LINE))
@@ -872,7 +878,7 @@ static const struct http_topic_exdata_compose g_topic_exdata_compose[HTTPD_TOPIC
         http_decoder_half_data_get_response_line(res_data, line);
     }
 
-    void http_message_get_header(const struct http_message *msg, const hstring *key,
+    void http_message_header_get0(const struct http_message *msg, const hstring *key,
                                 struct http_header *hdr_result)
     {
         int ret = -1;
@@ -954,7 +960,7 @@ static const struct http_topic_exdata_compose g_topic_exdata_compose[HTTPD_TOPIC
         return -1;
     }
 
-    void http_message_get_raw_body(const struct http_message *msg,
+    void http_message_raw_body_get0(const struct http_message *msg,
                                   hstring *body)
     {
         int ret = -1;
@@ -984,7 +990,7 @@ static const struct http_topic_exdata_compose g_topic_exdata_compose[HTTPD_TOPIC
         return;
     }
 
-    void http_message_get_decompress_body(const struct http_message *msg,
+    void http_message_decompress_body_get0(const struct http_message *msg,
                                          hstring *body)
     {
         int ret = -1;
@@ -1014,7 +1020,7 @@ static const struct http_topic_exdata_compose g_topic_exdata_compose[HTTPD_TOPIC
         return;    
     }
 
-    void http_message_get_url(const struct http_message *msg, hstring *url)
+    void http_message_raw_url_get0(const struct http_message *msg, hstring *url)
     {
         if (unlikely(NULL == msg))
         {
@@ -1043,6 +1049,35 @@ static const struct http_topic_exdata_compose g_topic_exdata_compose[HTTPD_TOPIC
         return;    
     }
 
+    void http_message_decoded_url_get0(const struct http_message *msg, struct iovec *url)
+    {
+        if (unlikely(NULL == msg))
+        {
+            if(url){
+                url->iov_base = NULL;
+                url->iov_len = 0;
+            }
+            return;
+        }
+        assert(msg->ref_queue);
+        assert(msg->queue_index < HD_RESULT_QUEUE_LEN);
+
+        struct http_decoder_half_data *req_data =
+            msg->ref_queue->array[msg->queue_index].req_data;
+
+        if(http_half_data_get_decode_url(req_data, url) < 0){
+            goto fail;
+        }
+        return;
+
+    fail:
+        if(url){
+            url->iov_base = NULL;
+            url->iov_len = 0;
+        }
+        return;   
+    }
+
     int http_message_get_transaction_seq(const struct http_message *msg)
     {
         if (unlikely(NULL == msg))
diff --git a/src/http_decoder_half.cpp b/src/http_decoder_half.cpp
index b2d3f07..67fdb02 100644
--- a/src/http_decoder_half.cpp
+++ b/src/http_decoder_half.cpp
@@ -21,7 +21,9 @@ struct http_decoder_half_data
     size_t decompress_body_len;
 
     int joint_url_complete;
+    int url_is_encoded;
     hstring joint_url; // http://<host>[:<port>]/<path>?<searchpart>
+    hstring decoded_url;
     long long transaction_index;
 };
 
@@ -639,7 +641,7 @@ static void publish_message_for_parsed_header(struct http_decoder_half *half)
     return;
 }
 
-int http_decoder_half_parse(struct http_decoder_half *half, const char *data, size_t data_len)
+int http_decoder_half_parse(int proxy_enable, struct http_decoder_half *half, const char *data, size_t data_len)
 {
     assert(half && data);
 
@@ -659,7 +661,9 @@ int http_decoder_half_parse(struct http_decoder_half *half, const char *data, si
         llhttp_resume(&half->parser);
         break;
     case HPE_PAUSED_UPGRADE:
-        llhttp_resume_after_upgrade(&half->parser);
+        if(proxy_enable){
+            llhttp_resume_after_upgrade(&half->parser);
+        }
         ret = 0;
         break;
     default:
@@ -964,6 +968,15 @@ void http_decoder_join_url(struct http_decoder_half_data *hfdata, nmx_pool_t *me
     hfdata->joint_url.iov_base = url_cache_str;
     hfdata->joint_url.iov_len = url_cache_str_len;
 
+    hfdata->url_is_encoded = httpd_url_is_encoded(url_cache_str, url_cache_str_len);
+    if(hfdata->url_is_encoded){
+        hfdata->decoded_url.iov_base = MEMPOOL_CALLOC(mempool, char, url_cache_str_len);
+        httpd_url_decode(url_cache_str, url_cache_str_len, (char *)hfdata->decoded_url.iov_base, &hfdata->decoded_url.iov_len);
+    }else{
+        hfdata->decoded_url.iov_base = url_cache_str;
+        hfdata->decoded_url.iov_len = url_cache_str_len;
+    }
+
     hfdata->joint_url_complete = 1;
 }
 
@@ -1029,6 +1042,16 @@ int http_half_data_get_url(struct http_decoder_half_data *res_data, hstring *url
     url->iov_len = res_data->joint_url.iov_len;
     return 0;
 }
+int http_half_data_get_decode_url(struct http_decoder_half_data *res_data, hstring *url)
+{
+    if (0 == res_data->joint_url_complete)
+    {
+        return -1;
+    }
+    url->iov_base = res_data->decoded_url.iov_base;
+    url->iov_len = res_data->decoded_url.iov_len;
+    return 0;
+}
 
 int http_half_data_get_transaction_seq(struct http_decoder_half_data *hf_data)
 {
@@ -1056,7 +1079,7 @@ void http_half_pre_context_free(struct session *sess, struct http_decoder_exdata
         res_data = queue->array[i].res_data;
         if ((req_data != NULL) && (NULL == res_data) && (req_data->state < HTTP_EVENT_REQ_END))
         {
-            msg = http_message_new(HTTP_TRANSACTION_FREE, queue, i, HTTP_REQUEST);
+            msg = http_message_new(HTTP_TRANSACTION_END, queue, i, HTTP_REQUEST);
             session_mq_publish_message(sess, exdata->pub_topic_id, msg);
         }
     }
@@ -1065,7 +1088,7 @@ void http_half_pre_context_free(struct session *sess, struct http_decoder_exdata
         res_data = queue->array[i].res_data;
         if ((res_data != NULL) && (res_data->state < HTTP_EVENT_RES_END))
         {
-            msg = http_message_new(HTTP_TRANSACTION_FREE, queue, i, HTTP_RESPONSE);
+            msg = http_message_new(HTTP_TRANSACTION_END, queue, i, HTTP_RESPONSE);
             session_mq_publish_message(sess, exdata->pub_topic_id, msg);
         }
     }
diff --git a/src/http_decoder_half.h b/src/http_decoder_half.h
index 5183171..9ea5169 100644
--- a/src/http_decoder_half.h
+++ b/src/http_decoder_half.h
@@ -52,7 +52,7 @@ void http_decoder_half_reinit(struct http_decoder_half *half,
                               struct http_decoder_result_queue *queue,
                               nmx_pool_t *mempool, struct session *sess);
 
-int http_decoder_half_parse(struct http_decoder_half *half, const char *data, size_t data_len);
+int http_decoder_half_parse(int proxy_enable, struct http_decoder_half *half, const char *data, size_t data_len);
 
 long long http_decoder_half_trans_count(struct http_decoder_half *half);
 
@@ -84,6 +84,7 @@ void http_decoder_half_data_dump(struct http_decoder_half *half);
 
 void http_decoder_get_host_feed_url(struct http_decoder_half *half);
 void http_decoder_get_url(struct http_decoder_half_data *hfdata, nmx_pool_t *mempool);
+int http_half_data_get_decode_url(struct http_decoder_half_data *res_data, hstring *url);
 void http_decoder_join_url(struct http_decoder_half_data *hfdata,
                            nmx_pool_t *mempool,
                            const struct http_header *host_hdr);
diff --git a/src/http_decoder_inc.h b/src/http_decoder_inc.h
index 13f0369..aa213c3 100644
--- a/src/http_decoder_inc.h
+++ b/src/http_decoder_inc.h
@@ -16,6 +16,7 @@
 extern "C"
 {
 #endif
+#include <bits/types/struct_iovec.h>
 #include "stellar/stellar.h"
 #include "stellar/utils.h"
 #include "stellar/session.h"
@@ -63,7 +64,7 @@ extern "C"
 #define DEFAULT_MEMPOOL_SIZE (32 * 1024)
 
 #define HTTPD_CFG_FILE   "./etc/http/http_decoder.toml"
-#define FILEDSTAT_OUTPUT_FILE   "./http_decoder.fs4"
+#define FILEDSTAT_OUTPUT_FILE   "./metrics/http_decoder_fs4.json"
 
 #define HTTP_CTX_NOT_HTTP "__NOT_HTTP_SESS__"
 #define HTTP_CTX_IS_HTTP  "__FAKE_HTTP_CTX__"
@@ -73,6 +74,7 @@ struct http_decoder_config
     int decompress_switch;
     int stat_interval_pkts; // call fieldstat_incrby every stat_interval_pkts
     int stat_output_interval;
+    int proxy_enable;
     size_t result_queue_len; // per session result queue length
     size_t mempool_size;     // per session mempool size
 };
diff --git a/src/http_decoder_stat.cpp b/src/http_decoder_stat.cpp
index 290af64..160353b 100644
--- a/src/http_decoder_stat.cpp
+++ b/src/http_decoder_stat.cpp
@@ -28,12 +28,9 @@ static const struct hd_stat_config_tuple g_httpd_stat_tuple[HTTPD_STAT_MAX] =
 
 void http_decoder_stat_free(struct http_decoder_stat *hd_stat)
 {
-    pthread_cancel(hd_stat->timer_pid);
-    void *join_res = NULL;
-    do{
-        pthread_join(hd_stat->timer_pid, &join_res);
-    }while(join_res != PTHREAD_CANCELED);
-
+    if(hd_stat->timer_pid != 0){
+        pthread_cancel(hd_stat->timer_pid);
+    }
     if(hd_stat->stats != NULL){
         free(hd_stat->stats);
     }
@@ -71,22 +68,25 @@ int http_decoder_stat_init(struct http_decoder_stat *hd_stat, int thread_max, in
         if (hd_stat->field_stat_id[i] < 0)
         {
             fprintf(stderr, "fieldstat_easy_register_counter %s failed.", g_httpd_stat_tuple[i].name);
+            fieldstat_easy_free(hd_stat->fse);
+            hd_stat->fse = NULL;
             return -1;
         }
     }
-
-    hd_stat->stats = (struct hd_statistics *)calloc(thread_max, sizeof(struct hd_statistics));
-    hd_stat->stat_interval_pkts = stat_interval_pkts;
    
     int ret = fieldstat_easy_enable_auto_output(hd_stat->fse, FILEDSTAT_OUTPUT_FILE, stat_interval_time);
     if (ret < 0)
     {
         fprintf(stderr, "fieldstat_easy_enable_auto_output failed.");
+        fieldstat_easy_free(hd_stat->fse);
+        hd_stat->fse = NULL;        
         return -1;
     }
-
+    hd_stat->stats = (struct hd_statistics *)calloc(thread_max, sizeof(struct hd_statistics));
+    hd_stat->stat_interval_pkts = stat_interval_pkts;
+    hd_stat->stat_interval_time = stat_interval_time;
     pthread_create(&hd_stat->timer_pid, NULL, httpd_stat_timer_thread, hd_stat);
-
+    pthread_detach(hd_stat->timer_pid);
     return 0;
 }
 
@@ -96,6 +96,10 @@ void http_decoder_stat_update(struct http_decoder_stat *hd_stat, int thread_id,
     assert(thread_id >= 0);
     assert(type < HTTPD_STAT_MAX);
 
+    if(unlikely(hd_stat->stats == NULL)){
+        return;
+    }
+
     struct hd_statistics *cur_hds = &hd_stat->stats[thread_id];
 
     cur_hds->counter[type] += value;
diff --git a/src/http_decoder_stat.h b/src/http_decoder_stat.h
index f78f6b6..0376d95 100644
--- a/src/http_decoder_stat.h
+++ b/src/http_decoder_stat.h
@@ -44,6 +44,7 @@ struct http_decoder_stat
     long long current_time_ms;
     struct fieldstat_easy *fse;
     int stat_interval_pkts; // call fieldstat_incrby every stat_interval_pkts
+    int stat_interval_time; //second
     int field_stat_id[HTTPD_STAT_MAX];
     struct hd_statistics *stats; //size is thread number    
 };
diff --git a/src/http_decoder_tunnel.cpp b/src/http_decoder_tunnel.cpp
index 67445f3..9cf477d 100644
--- a/src/http_decoder_tunnel.cpp
+++ b/src/http_decoder_tunnel.cpp
@@ -13,8 +13,12 @@ struct http_tunnel_message
 };
 
 
-int httpd_tunnel_identify(int curdir, struct http_decoder_half_data *hfdata)
+int httpd_tunnel_identify(struct http_decoder_env *httpd_env, int curdir, struct http_decoder_half_data *hfdata)
 {
+    if(0 == httpd_env->hd_cfg.proxy_enable){
+        return 0;
+    }
+
     if(PACKET_DIRECTION_C2S == curdir){
         struct http_request_line reqline = {};
         http_decoder_half_data_get_request_line(hfdata, &reqline);
diff --git a/src/http_decoder_tunnel.h b/src/http_decoder_tunnel.h
index 2b67649..f908c1b 100644
--- a/src/http_decoder_tunnel.h
+++ b/src/http_decoder_tunnel.h
@@ -1,7 +1,6 @@
 #pragma once 
-
+#include "http_decoder_inc.h"
 #include "http_decoder_half.h"
-#include "llhttp.h"
 
 enum http_tunnel_state{
     HTTP_TUN_NON = 0,       // init, or not tunnel session
@@ -12,7 +11,23 @@ enum http_tunnel_state{
     HTTP_TUN_INNER_TRANS,   // http inner tunnel protocol transmitting 
 };
 
-int httpd_tunnel_identify(int curdir, struct http_decoder_half_data *hfdata);
+/************************************************************
+*       HTTP TUNNEL WITH CONNECT METHOD.
+*************************************************************/
+struct http_tunnel_message;
+#define HTTP_DECODER_TUNNEL_TOPIC   "HTTP_DECODER_TUNNEL_MESSAGE"
+
+enum http_tunnel_message_type {
+    HTTP_TUNNEL_OPENING,
+    HTTP_TUNNEL_ACTIVE,
+    HTTP_TUNNEL_CLOSING,
+    HTTP_TUNNEL_MSG_MAX
+};
+enum http_tunnel_message_type http_tunnel_message_type_get(const struct http_tunnel_message *tmsg);
+void http_tunnel_message_get_payload(const struct http_tunnel_message *tmsg, struct iovec *tunnel_payload);
+
+
+int httpd_tunnel_identify(struct http_decoder_env *httpd_env, int curdir, struct http_decoder_half_data *hfdata);
 int httpd_is_tunnel_session(const struct http_decoder_exdata *ex_data);
 int httpd_in_tunnel_transmitting(struct http_decoder_exdata *ex_data);
 void httpd_tunnel_state_update(struct http_decoder_exdata *ex_data);
diff --git a/src/http_decoder_utils.cpp b/src/http_decoder_utils.cpp
index 6d71bdb..b2af0fc 100644
--- a/src/http_decoder_utils.cpp
+++ b/src/http_decoder_utils.cpp
@@ -1,5 +1,6 @@
 #include <string.h>
 #include <assert.h>
+#include "http_decoder.h"
 #include "http_decoder_inc.h"
 
 char *safe_dup(const char *str, size_t len)
@@ -38,6 +39,9 @@ const char *http_message_type_to_string(enum http_message_type type)
     case HTTP_MESSAGE_REQ_HEADER_END:
         sname = "HTTP_MESSAGE_REQ_HEADER_END";
         break;
+    case HTTP_MESSAGE_REQ_BODY_START:
+        sname = "HTTP_MESSAGE_REQ_BODY_START";
+        break;  
     case HTTP_MESSAGE_REQ_BODY: 
         sname = "HTTP_MESSAGE_REQ_BODY";
         break;
@@ -53,17 +57,20 @@ const char *http_message_type_to_string(enum http_message_type type)
     case HTTP_MESSAGE_RES_HEADER_END:
         sname = "HTTP_MESSAGE_RES_HEADER_END";
         break;
+    case HTTP_MESSAGE_RES_BODY_START:
+        sname = "HTTP_MESSAGE_RES_BODY_START";
+        break;  
     case HTTP_MESSAGE_RES_BODY:
         sname = "HTTP_MESSAGE_RES_BODY";
         break;  
     case HTTP_MESSAGE_RES_BODY_END:
         sname = "HTTP_MESSAGE_RES_BODY_END";
         break;
-    case HTTP_TRANSACTION_NEW:
-        sname = "HTTP_TRANSACTION_NEW";
+    case HTTP_TRANSACTION_START:
+        sname = "HTTP_TRANSACTION_START";
         break;
-    case HTTP_TRANSACTION_FREE:
-        sname = "HTTP_TRANSACTION_FREE";
+    case HTTP_TRANSACTION_END:
+        sname = "HTTP_TRANSACTION_END";
         break;
 
     default:
@@ -81,6 +88,7 @@ int http_message_type_is_req(struct session *sess, enum http_message_type msg_ty
         case HTTP_MESSAGE_REQ_LINE:
         case HTTP_MESSAGE_REQ_HEADER:
         case HTTP_MESSAGE_REQ_HEADER_END:
+        case HTTP_MESSAGE_REQ_BODY_START:
         case HTTP_MESSAGE_REQ_BODY:
         case HTTP_MESSAGE_REQ_BODY_END:
             is_req_msg = 1;
@@ -89,13 +97,14 @@ int http_message_type_is_req(struct session *sess, enum http_message_type msg_ty
         case HTTP_MESSAGE_RES_LINE:
         case HTTP_MESSAGE_RES_HEADER:
         case HTTP_MESSAGE_RES_HEADER_END:
+        case HTTP_MESSAGE_RES_BODY_START:
         case HTTP_MESSAGE_RES_BODY:
         case HTTP_MESSAGE_RES_BODY_END:
             is_req_msg = 0;
         break;
 
-        case HTTP_TRANSACTION_NEW:
-        case HTTP_TRANSACTION_FREE:
+        case HTTP_TRANSACTION_START:
+        case HTTP_TRANSACTION_END:
         {
             int cur_dir = packet_get_direction(session_get0_current_packet(sess));
             if(PACKET_DIRECTION_C2S == cur_dir){
@@ -154,4 +163,58 @@ int stellar_session_mq_get_topic_id_reliable(struct stellar *st, const char *top
         topic_id = stellar_session_mq_create_topic(st, topic_name, msg_free_cb, msg_free_arg);
     }
     return topic_id;
+}
+
+static const unsigned char __g_httpd_hextable[] = {
+  0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0,       /* 0x30 - 0x3f */
+  0, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x40 - 0x4f */
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,       /* 0x50 - 0x5f */
+  0, 10, 11, 12, 13, 14, 15                                      /* 0x60 - 0x66 */
+};
+
+/* the input is a single hex digit */
+#define onehex2dec(x) __g_httpd_hextable[x - '0']
+
+#include <ctype.h>
+// https://github.com/curl/curl/blob/2e930c333658725657b94a923d175c6622e0f41d/lib/urlapi.c
+void httpd_url_decode(const char *string, size_t length, char *ostring, size_t *olen)
+{
+    size_t alloc = length;
+    char *ns = ostring;
+
+    while (alloc)
+    {
+        unsigned char in = (unsigned char)*string;
+        if (('%' == in) && (alloc > 2) &&
+            isxdigit(string[1]) && isxdigit(string[2]))
+        {
+            /* this is two hexadecimal digits following a '%' */
+            in = (unsigned char)(onehex2dec(string[1]) << 4) | onehex2dec(string[2]);
+            string += 3;
+            alloc -= 3;
+        }
+        else
+        {
+            string++;
+            alloc--;
+        }
+        *ns++ = (char)in;
+    }
+    *ns = 0; /* terminate it */
+
+    if (olen)
+        /* store output size */
+        *olen = ns - ostring;
+
+    return;
+}
+
+int httpd_url_is_encoded(const char *url, size_t len)
+{
+    for(size_t i = 0; i < len; i++){
+        if(url[i] == '%'){
+            return 1;
+        }
+    }
+    return 0;
 }
\ No newline at end of file
diff --git a/src/http_decoder_utils.h b/src/http_decoder_utils.h
index 5b09d50..b3df52d 100644
--- a/src/http_decoder_utils.h
+++ b/src/http_decoder_utils.h
@@ -1,8 +1,21 @@
-#ifndef _HTTP_DECODER_UTILS_H_
-#define _HTTP_DECODER_UTILS_H_
+#pragma once 
 
 #include <stdlib.h>
 #include <stdio.h>
+#include <cstddef>
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+#include <bits/types/struct_iovec.h>
+#include "stellar/stellar.h"
+#include "stellar/utils.h"
+#include "stellar/session.h"
+#include "stellar/session_mq.h"
+#include "stellar/session_exdata.h"
+#ifdef __cplusplus
+}
+#endif
 
 char *safe_dup(const char *str, size_t len);
 int strncasecmp_safe(const char *fix_s1, const char *dyn_s2, size_t fix_n1, size_t dyn_n2);
@@ -10,6 +23,8 @@ const char *http_message_type_to_string(enum http_message_type type);
 int http_message_type_is_req(struct session *sess, enum http_message_type msg_type);
 int http_event_is_req(enum http_event event);
 int stellar_session_mq_get_topic_id_reliable(struct stellar *st, const char *topic_name, session_msg_free_cb_func *msg_free_cb, void *msg_free_arg);
+void httpd_url_decode(const char *string, size_t length, char *ostring, size_t *olen);
+int httpd_url_is_encoded(const char *url, size_t len);
 /******************************************************************************
  * Logger
  ******************************************************************************/
@@ -45,4 +60,3 @@ enum http_decoder_log_level {
         }                                                                        \
     }
 #endif
-#endif
\ No newline at end of file