Feature: HTTP unzip content is consistent with the packet life cyclev2.0.18

4 files changed, 56 insertions, 45 deletions

diff --git a/.gitignore b/.gitignore
index 6955643..bbf96e4 100755
--- a/.gitignore
+++ b/.gitignore
@@ -4,7 +4,7 @@ SI/
 *.so
 *.si4project/
 *.d
-build/
+build*/
 .vscode
 .idea
 core.*
diff --git a/src/HTTP_Analyze.c b/src/HTTP_Analyze.c
index e7bae9b..3be9c8d 100644
--- a/src/HTTP_Analyze.c
+++ b/src/HTTP_Analyze.c
@@ -123,8 +123,8 @@ void history()
 	//2019-01-05 V4.0 //special http packet, get_span_space.pcap 	
 	//2019-01-11 V4.0 //is_absolute_uri 
 	//2019-01-28 V4.0 //1. support Content-Range: bytes -154100106/154100107 2. bug:http_doWithGzipData 3. assert(session_info.buflen>=0);
-	//2019-11-12 V4.0 //����http_host_parser����
-	//2019-12-05 V4.0 //����http_line2value����
+	//2019-11-12 V4.0 //����http_host_parser����
+	//2019-12-05 V4.0 //����http_line2value����
 	//2020-01-08 V4.0 //add mesa_proto
 	//2020-06-12 V4.0 //set pbuf=null when resethttpstream
 	//2020-07-22 V4.0 //range Invalid
@@ -299,7 +299,7 @@ void http_callPluginBatch(http_parser_t* cur_http_node,struct streaminfo *a_tcp,
 }
 
 
-/*�ֶλص���ͬʱ��������ֵ*/
+/*�ֶλص���ͬʱ��������ֵ*/
 void http_callPluginPreFieldBatch(http_parser_t* cur_http_node, struct streaminfo *a_tcp, int thread_seq, void *a_packet)
 {
 	if(HTTP_INTEREST_KEY_MASK!=cur_http_node->interested_reg_mask && 
@@ -354,7 +354,7 @@ void http_callPluginPreFieldBatch(http_parser_t* cur_http_node, struct streaminf
 	}		
 }
 
-/*�������ص�*/
+/*�������ص�*/
 void http_callPluginPreBatch(http_parser_t* cur_http_node, struct streaminfo *a_tcp, int thread_seq, void *a_packet)
 {
 	if(HTTP_INTEREST_KEY_MASK!=cur_http_node->interested_reg_mask) 
@@ -422,7 +422,7 @@ void http_callPluginPreBatch(http_parser_t* cur_http_node, struct streaminfo *a_
 	}		
 }
 
-/*�ֶλص���ͬʱ��������ֵ*/
+/*�ֶλص���ͬʱ��������ֵ*/
 void http_callPlugin(http_parser_t* cur_http_node, struct streaminfo *a_tcp, int thread_seq, void *a_packet)
 {
 	int64 	prot_flag = (((int64)1)<<cur_http_node->interested_reg_mask);	
@@ -487,7 +487,7 @@ void http_callPlugin(http_parser_t* cur_http_node, struct streaminfo *a_tcp, int
 #endif
 }
 
-/*�ֶλص����������ص��ֿ�������*/
+/*�ֶλص����������ص��ֿ�������*/
 void http_callPlugin_field_or_batch(http_parser_t* cur_http_node, struct streaminfo *a_tcp, int thread_seq, void *a_packet)
 {
 	int64 	prot_flag = (((int64)1)<<cur_http_node->interested_reg_mask);	
@@ -688,6 +688,13 @@ void http_releaseHttpLinkNode(http_stream *a_http_stream,struct streaminfo *a_tc
 			dictator_free(thread_seq, cur_node->parser.batch_infor);
 		}	
 	}
+
+	if(cur_node->last_unzip_content.buf!=NULL)
+	{
+		dictator_free(thread_seq, cur_node->last_unzip_content.buf);
+		cur_node->last_unzip_content.buf = NULL;
+		cur_node->last_unzip_content.buflen = 0;
+	}
 		
 	if(cur_node->ungzip_handle!=NULL)
 	{	
@@ -844,7 +851,7 @@ void http_initHttpStream(http_stream**pme, int thread_seq)
 void http_prevAnylse(http_parser_t* cur_node, http_stream* a_http_stream, int thread_seq)
 {
 	cur_node->processed_offset = a_http_stream->packet_offset;	
-	/*20181228 �ϸ�����buf���ݸ���http_info*/
+	/*20181228 �ϸ�����buf���ݸ���http_info*/
 	if(a_http_stream->buflen>0)
 	{
 		if(NULL==cur_node->pbuf)
@@ -923,7 +930,7 @@ uchar http_judgeHttpProtocol(http_stream *a_http_stream, uchar curdir, struct tc
 	uchar 	method = HTTP_METHOD_UNKNOWN;
 	uchar  	buf[HTTP_START_FLAGS_LEN] = {0};
 	
-	//��һ������ȥ�����У��ո��Ʊ�������HTTP_START_FLAGS_LEN�ֽ�
+	//��һ������ȥ�����У��ո��Ʊ�������HTTP_START_FLAGS_LEN�ֽ�
 	if(0==*buflen)
 	{
 		if(tcp_detail->datalen < HTTP_START_FLAGS_LEN+*offset)
@@ -939,7 +946,7 @@ uchar http_judgeHttpProtocol(http_stream *a_http_stream, uchar curdir, struct tc
 		return http_judgeHttpMethod(&method, (char*)tcp_detail->pdata+*offset, curdir); 	
 	}
 	
-	//һֱ�ȹ�HTTP_START_FLAGS_LEN�ֽ�
+	//һֱ�ȹ�HTTP_START_FLAGS_LEN�ֽ�
 	if(0<*buflen)
 	{
 		if(tcp_detail->datalen+*buflen < HTTP_START_FLAGS_LEN+*offset && tcp_detail->datalen-*offset>0)
@@ -952,7 +959,7 @@ uchar http_judgeHttpProtocol(http_stream *a_http_stream, uchar curdir, struct tc
 			}			
 			return GO_BACK;			
 		}
-		//��־һ������buffer���棬һ������data����		
+		//��־һ������buffer���棬һ������data����		
 		/*20181113*/		
 		//a_http_stream->pbuf = (char*)dictator_realloc(thread_seq, a_http_stream->pbuf, HTTP_START_FLAGS_LEN);
 		//memcpy((void*)(a_http_stream->pbuf+(*buflen)), (void*)((uchar*)tcp_detail->pdata+*offset),HTTP_START_FLAGS_LEN-*buflen);
@@ -983,7 +990,7 @@ uchar http_analyseHttpConnection(http_stream* a_http_stream,struct streaminfo *a
 	/*http identify : v3 judge is http stream or not firstly*/
 	if(HTTP_FLASE==a_http_stream->maybe_http_stream)
 	{
-		/*20190105:���󷽷�֮ǰ�Ŀո�ɾ������Ӱ��ʶ���Ѿ���ʼ��������֮����Ҫɾ���ո������޷�������ʼ�е�����Ԫ��*/
+		/*20190105:���󷽷�֮ǰ�Ŀո�ɾ������Ӱ��ʶ���Ѿ���ʼ��������֮����Ҫɾ���ո������޷�������ʼ�е�����Ԫ��*/
 		if(a_http_stream->pbuf==NULL)
 		{
 			http_deleteEmptyRow(&(a_http_stream->packet_offset), (char*)(tcp_detail->pdata), tcp_detail->datalen);
@@ -1028,7 +1035,7 @@ uchar http_analyseHttpConnection(http_stream* a_http_stream,struct streaminfo *a
 #endif
 */
 
-	/*http lost : lostlen must be proc befor http_initHttpConnection���൱��֮ǰ������*/
+	/*http lost : lostlen must be proc befor http_initHttpConnection���൱��֮ǰ������*/
 	if(tcp_detail->lostlen>0)
 	{			
 		if(g_http_prog_para.http_stat_cycle)
@@ -1054,12 +1061,12 @@ uchar http_analyseHttpConnection(http_stream* a_http_stream,struct streaminfo *a
 	/*avoid  keep HTTP_RETURN_DROPPKT*/
 	cur_http_node->phttp_begin = (char*)tcp_detail->pdata+cur_http_node->processed_offset;
 
-	//ͷ������
+	//ͷ������
 	rec = http_analyseHttpReqResHeader(cur_http_node, a_http_stream, a_tcp, thread_seq, a_packet);
 	switch(cur_http_node->mgs_status)
 	{
 		case HTTP_RETURN_RESET:
-			if(a_http_stream->is_http_falgs==HTTP_FLASE)//ȷ������httpЭ��
+			if(a_http_stream->is_http_falgs==HTTP_FLASE)//ȷ������httpЭ��
 			{					
 				return APP_STATE_DROPME;
 			}
@@ -1105,7 +1112,7 @@ uchar http_analyseHttpConnection(http_stream* a_http_stream,struct streaminfo *a
 			}
 		}
 		/*batch mode, callback, span pkt ,callback.   cur_http_node->parser.curdir!=0 when start line*/
-		/*������ֻ����*/
+		/*������ֻ����*/
 		//if(HTTP_CALLBACK_MODE_BATCH==g_http_prog_para.callback_mode || g_http_prog_para.need_http_state )
 		if((HTTP_CALLBACK_MODE_BATCH==g_http_prog_para.callback_mode || g_http_prog_para.need_http_state)&& cur_http_node->parser.curdir!=0 )
 		{
@@ -1133,7 +1140,7 @@ uchar http_analyseHttpConnection(http_stream* a_http_stream,struct streaminfo *a
 #endif
 */
 
-	//��Ϣ�����
+	//��Ϣ�����
 	/*set mgs_status because cur_http_node maybe release*/
 	uchar mgs_status = HTTP_RETURN_GIVEME;
 	rec = http_findAndDoWithEntity(&mgs_status, cur_http_node, a_http_stream, a_tcp, thread_seq, a_packet);
diff --git a/src/HTTP_Message.h b/src/HTTP_Message.h
index 18b65d1..ca901ab 100644
--- a/src/HTTP_Message.h
+++ b/src/HTTP_Message.h
@@ -168,6 +168,8 @@ typedef struct _http_parser_t
 	char						endline_flag;  		        //complete line flag: \r\n, \r, \n.
 	uint8 						mgs_status;					//return state							
 	uint8						header_span_flag;			//see define 
+
+	fold_infor_t last_unzip_content;
 		
 #if HTTP_PROXY
 	uchar						proxy_flag;					//DIR_C2S, DIR_S2C,DIR_DOUBLE	
diff --git a/src/HTTP_Message_Entry.c b/src/HTTP_Message_Entry.c
index 6de72ec..1c51735 100644
--- a/src/HTTP_Message_Entry.c
+++ b/src/HTTP_Message_Entry.c
@@ -41,7 +41,7 @@ static void http_clearSpace(http_stream *a_http_stream, struct streaminfo *a_tcp
 
 void http_doWithGzipData(http_parser_t *cur_http_node, struct streaminfo *a_tcp, int thread_seq, void *a_packet)
 {
-	int		ret = 0;
+	int	ret=0;
 	result_array_t *result_array = (result_array_t*)dictator_malloc(thread_seq, sizeof(result_array_t));
 	memset(result_array, 0, sizeof(result_array_t));
 	if(cur_http_node->ungzip_handle==NULL)
@@ -68,35 +68,37 @@ void http_doWithGzipData(http_parser_t *cur_http_node, struct streaminfo *a_tcp,
 		dictator_free(thread_seq, result_array);
 		return ;
 	}
-	ret = docanalyze_parsestream(cur_http_node->ungzip_handle,
+
+	cur_http_node->last_unzip_content.buflen=0;
+	if(cur_http_node->last_unzip_content.buf!=NULL)
+	{
+		dictator_free(thread_seq, cur_http_node->last_unzip_content.buf);
+		cur_http_node->last_unzip_content.buf=NULL;
+	}
+
+	ret=docanalyze_parsestream(cur_http_node->ungzip_handle,
 								(const char*)cur_http_node->session.buf, 
 								cur_http_node->session.buflen,
 								result_array);
 	if(ret==DOC_PRO_OK)
 	{
-		cur_http_node->parser.append_infor.content = cur_http_node->session.buf;
-		cur_http_node->parser.append_infor.contlen = cur_http_node->session.buflen;
-		cur_http_node->session.buf =NULL;
-		cur_http_node->session.buflen = 0;
 		int k=0;
 		for (k=0; k < result_array->result_num; k++)
 		{
-			cur_http_node->session.buf = (char*)dictator_realloc(thread_seq, cur_http_node->session.buf,cur_http_node->session.buflen+result_array->result_buff[k].size);
-			memcpy(cur_http_node->session.buf+cur_http_node->session.buflen,
+			cur_http_node->last_unzip_content.buf=(char*)dictator_realloc(thread_seq, cur_http_node->last_unzip_content.buf, cur_http_node->last_unzip_content.buflen+result_array->result_buff[k].size);
+			memcpy(cur_http_node->last_unzip_content.buf+cur_http_node->last_unzip_content.buflen,
 				     result_array->result_buff[k].presult,
 				     result_array->result_buff[k].size);
-			cur_http_node->session.buflen += result_array->result_buff[k].size;     		 	
+			cur_http_node->last_unzip_content.buflen += result_array->result_buff[k].size;     		 	
 		}
+
 		FLAG_SET(cur_http_node->flag, HTTP_FLAG_BATCH_CALLBACK);
+
+		cur_http_node->session.buf=cur_http_node->last_unzip_content.buf;
+		cur_http_node->session.buflen=cur_http_node->last_unzip_content.buflen;
 		http_callPlugin(cur_http_node, a_tcp, thread_seq, a_packet);
-		if(cur_http_node->session.buf!=NULL)
-		{
-			dictator_free(thread_seq, cur_http_node->session.buf);
-			cur_http_node->session.buf = NULL;	
-			cur_http_node->session.buflen = 0;			
-		}
-		cur_http_node->parser.append_infor.content = NULL;
-		cur_http_node->parser.append_infor.contlen = 0;
+		cur_http_node->session.buf=NULL;	
+		cur_http_node->session.buflen=0;
 	}
 	else if(ret==DOC_PRO_ERR)
 	{
@@ -251,7 +253,7 @@ uchar http_doWithProxyData(http_parser_t *a_http, http_stream *a_http_stream, st
 		*offset += len;			
 		rec = GO_BACK;
 	}	
-	/*����һ��pkt��ʼ�ص��������������������HTTP�����������ѭ��*/
+	/*����һ��pkt��ʼ�ص��������������������HTTP�����������ѭ��*/
 	if(len==0)
 	{
 		a_http->proxy_cb_flag = 1;
@@ -265,11 +267,11 @@ uchar http_doWithProxyData(http_parser_t *a_http, http_stream *a_http_stream, st
 
 
 /**********************************************************
- * ���ܣ�Ѱ��chunked�����ݿ�ʼλ��.
- * a_http_node:��Ϣ�ڵ�,�������������Ҳ�Ƿ��ز�����
- * cur_half����ǰ���ݡ�
- * ע��after_cr_data_len<1����û�ҵ���ʼλ��
- *   		 (*a_http_node)->crlf_offset<1����û�õ����ȡ�
+ * ���ܣ�Ѱ��chunked�����ݿ�ʼλ��.
+ * a_http_node:��Ϣ�ڵ�,�������������Ҳ�Ƿ��ز�����
+ * cur_half����ǰ���ݡ�
+ * ע��after_cr_data_len<1����û�ҵ���ʼλ��
+ *   		 (*a_http_node)->crlf_offset<1����û�õ����ȡ�
  **********************************************************/
 uchar http_findChunkedDataStartPos(http_parser_t *a_http_node, uchar *tcp_data, UINT32 tcp_data_len)
 {
@@ -338,10 +340,10 @@ uchar http_readChunkedData(http_parser_t *a_http, struct streaminfo *a_tcp, int
 }
 
 /**********************************************************
- * ���ܣ���ȡchunked�ij���.
- * a_http_node:��Ϣ�ڵ�,�������������Ҳ�Ƿ��ز�����
- * chunked_len:���ֵ��chunked���ݳ���.
- * cur_half����ǰ���ݡ�
+ * ���ܣ���ȡchunked�ij���.
+ * a_http_node:��Ϣ�ڵ�,�������������Ҳ�Ƿ��ز�����
+ * chunked_len:���ֵ��chunked���ݳ���.
+ * cur_half����ǰ���ݡ�
  * return : HTTP_RETURN_SPAN_PACKET HTTP_RETURN_UNNORM
  **********************************************************/
 uchar http_readChunkedLength(struct streaminfo *a_tcp, http_parser_t *a_http, char *tcp_data, uint32 tcp_data_len,int thread_seq)
@@ -353,7 +355,7 @@ uchar http_readChunkedLength(struct streaminfo *a_tcp, http_parser_t *a_http, ch
 	char 	chunklen_buf[32] = {0};
 	uint32* offset = &a_http->processed_offset;
 
-	/*��һ��buf������*/
+	/*��һ��buf������*/
 	if(0==a_http->buflen)
 	{		
 		http_deleteEmptyRow_chunk(offset, tcp_data, tcp_data_len);