about hashtable

2 files changed, 66 insertions, 69 deletions

diff --git a/qq_file_send.c b/qq_file_send.c
index 031c9ac..cce54a4 100644
--- a/qq_file_send.c
+++ b/qq_file_send.c
@@ -1,5 +1,6 @@
 #include <time.h>
 #include <arpa/inet.h>
+#include <unistd.h>
 #include <stdio.h>
 #include <ctype.h>
 #include <MESA/MESA_prof_load.h>
@@ -13,7 +14,8 @@
 #define ONLINE_SEND 0
 #define OFFLINE_SEND 1
 #define NAME_SIZE 128
-#define MAX_HTABLE_NUM 24
+#define MAX_HTABLE_NUM 96
+#define UUID_LENGTH 35
 
 #define NIRVANA
 
@@ -23,17 +25,17 @@ void qq_file_send_version_1_20201124()
     //20201124 
 }
 
-
 typedef struct
 {
     uint8_t send_type; // OFFLINE_SEND
-    char uuid[36];
+    char uuid[UUID_LENGTH+1];
     uint16_t content_len;
     char filename[NAME_SIZE];
-    struct nirvana_streaming_asyctx *ctx; 
+    struct nirvana_streaming_ctx *ctx; 
     uint8_t next_flag;
     uint64_t file_len;
     uchar curdir;
+    uint id;
 }qq_pme_info,*qq_pme_info_p;
 
 typedef struct
@@ -52,12 +54,12 @@ typedef struct
 
 typedef struct
 {
+    uint id;
     char filename[NAME_SIZE];
-    uint64_t file_len;
-    struct nirvana_streaming_asyctx *ctx;
+    struct nirvana_streaming_ctx *ctx;
 }file_info;
 
-uint file_cnt = 0;
+uint capture_file_cnt = 0;
 
 struct_separator_feature online_feature,offline_feature;
 MESA_htable_handle file_htable;
@@ -65,7 +67,7 @@ char save_directory[NAME_SIZE];
 int locally_save;  // save file to local machine
 void *runtime_log;
 struct event_base *ev_base;
-struct nirvana_asyn_instance *instance_asyn;
+struct nirvana_instance *instance_asyn;
 
 /* 
  * @Description: use like strncmp
@@ -97,14 +99,14 @@ int bicmp(char *str,char *ptn,int len)
 */ 
 int parse_separator(char **payload,int payload_len,struct_separator_feature separator_feature,qq_pme_info **pme)
 {
-    int uuid_len = 35;  //this is a fixed value and will not change
+    int uuid_len = UUID_LENGTH; 
     //match by header
     if( bicmp(*payload,separator_feature.separator_header, separator_feature.header_len) != 0)
     {
         return -1;
     }
     // match by the suffix which is after uuid 
-    if( bicmp( (*payload)+separator_feature.uuid_deviation + uuid_len+1, separator_feature.uuid_suffix,separator_feature.uuid_suffix_len) != 0)
+    if( bicmp( (*payload)+separator_feature.uuid_deviation + uuid_len + 1, separator_feature.uuid_suffix,separator_feature.uuid_suffix_len) != 0)
     {
         return -2;
     }
@@ -128,15 +130,15 @@ int parse_separator(char **payload,int payload_len,struct_separator_feature sepa
  * @return: =0 : success
             <0 : fail    
 */ 
-int parse_end_suffix(char *payload,int payload_len,struct_separator_feature separator_feature)
+int parse_end_suffix(char *payload,int payload_len,struct_separator_feature separator_feature,uint file_id)
 {
 
     if( bicmp(separator_feature.end_suffix,payload,separator_feature.end_suffix_len) != 0)
     {
-        MESA_handle_runtime_log(runtime_log,RLOG_LV_DEBUG,"match","end suffix fail",payload_len);
+        MESA_handle_runtime_log(runtime_log,RLOG_LV_DEBUG,"match","file %u not finish",file_id);
         return -1;
     }
-    MESA_handle_runtime_log(runtime_log,RLOG_LV_DEBUG,"match","end suffix success",payload_len);
+    MESA_handle_runtime_log(runtime_log,RLOG_LV_DEBUG,"match","file %u finish",file_id);
     return 0;  
 }
 
@@ -196,7 +198,7 @@ int read_profile_of_separator(const char *inf_file,struct_separator_feature *p_s
     p_separator_feature->feature_type = feature_type;
     
     //header_len
-    read_res += MESA_load_profile_int_nodef(inf_file,section,"header_len",&(p_separator_feature->header_len));
+    read_res += MESA_load_profile_uint_nodef(inf_file,section,"header_len",&(p_separator_feature->header_len));
 
     //header
     p_separator_feature->separator_header = (char *)malloc(p_separator_feature->header_len+1);
@@ -210,10 +212,10 @@ int read_profile_of_separator(const char *inf_file,struct_separator_feature *p_s
     free(separator_header_array);
 
     //uuid_deviation
-    read_res += MESA_load_profile_int_nodef(inf_file,section,"uuid_deviation",&(p_separator_feature->uuid_deviation));
+    read_res += MESA_load_profile_uint_nodef(inf_file,section,"uuid_deviation",&(p_separator_feature->uuid_deviation));
 
     //uuid_suffix_len
-    read_res += MESA_load_profile_int_nodef(inf_file,section,"uuid_suffix_len",&(p_separator_feature->uuid_suffix_len));
+    read_res += MESA_load_profile_uint_nodef(inf_file,section,"uuid_suffix_len",&(p_separator_feature->uuid_suffix_len));
 
     //uuid_suffix 
     p_separator_feature->uuid_suffix = (char *)malloc(p_separator_feature->uuid_suffix_len+1);
@@ -227,10 +229,10 @@ int read_profile_of_separator(const char *inf_file,struct_separator_feature *p_s
     free(uuid_suffix_array);
 
     //content_deviation
-    read_res += MESA_load_profile_int_nodef(inf_file,section,"content_deviation",&(p_separator_feature->content_deviation));
+    read_res += MESA_load_profile_uint_nodef(inf_file,section,"content_deviation",&(p_separator_feature->content_deviation));
 
     //end_suffix_len
-    read_res += MESA_load_profile_int_nodef(inf_file,section,"end_suffix_len",&(p_separator_feature->end_suffix_len));
+    read_res += MESA_load_profile_uint_nodef(inf_file,section,"end_suffix_len",&(p_separator_feature->end_suffix_len));
 
     //end_suffix
     p_separator_feature->end_suffix = (char *)malloc(p_separator_feature->end_suffix_len+1);
@@ -241,7 +243,7 @@ int read_profile_of_separator(const char *inf_file,struct_separator_feature *p_s
         MESA_handle_runtime_log(runtime_log,RLOG_LV_FATAL,"init","[%s] end_suffix error",section);
     }
     str_to_ascii(end_suffix_array,p_separator_feature->end_suffix_len,p_separator_feature->end_suffix);
-    free(end_suffix_array);
+    free(end_suffix_array); 
     return read_res;
 }
 
@@ -254,6 +256,10 @@ void free_fileinfo(void *thisfile)
 {
     //thisfile = thisfile;
     file_info *p = (file_info *)thisfile;
+#ifdef NIRVANA
+    nirvana_streaming_update_end(p->ctx);
+#endif          
+    MESA_handle_runtime_log(runtime_log,RLOG_LV_DEBUG,"finish","close a file :%s",p->filename);
     free(p);
 }
 
@@ -268,22 +274,24 @@ void free_fileinfo(void *thisfile)
             0: not exist 
             1: exist
 */ 
-int match_uuid(qq_pme_info **pme,int content_len)
+int match_uuid(qq_pme_info **pme)
 {
     qq_pme_info_p qq_pme = *pme;
-    file_info* thisfile =  (file_info *) MESA_htable_search(file_htable,(u_char *)qq_pme->uuid,strlen(qq_pme->uuid));
+    file_info* thisfile =  (file_info *) MESA_htable_search(file_htable,(u_char *)qq_pme->uuid,UUID_LENGTH);
     if (thisfile == NULL)
     {
         // new file create
         file_info *newfile;
         newfile = (file_info *)malloc(sizeof(file_info));
-        sprintf(qq_pme->filename,"%s/%d-%s",save_directory,file_cnt,qq_pme->uuid);
-        file_cnt++;
+        sprintf(qq_pme->filename,"%s/%d-%s",save_directory,capture_file_cnt,qq_pme->uuid);
+        qq_pme->id = capture_file_cnt;
+        newfile->id = capture_file_cnt;
+        capture_file_cnt++;
         strncpy(newfile->filename,qq_pme->filename,strlen(qq_pme->filename));
-        newfile->file_len = content_len;
+        qq_pme->file_len = 0;
 #ifdef NIRVANA
         // nirvana section
-        struct nirvana_streaming_asyctx *ctx;
+        struct nirvana_streaming_ctx *ctx;
         union nirvana_progid progid;
         struct streaming_meta streammeta;
         struct nvn_opt_unit opt[1];
@@ -304,30 +312,27 @@ int match_uuid(qq_pme_info **pme,int content_len)
 
         calculate_md5_for_progid(newfile->filename, strlen(newfile->filename), &progid);
         streammeta.balance_seed = caculate_seed_using_sdbm_hash((char*)&progid, sizeof(progid));
-        ctx = nirvana_streaming_asyn_update_start(instance_asyn, &progid, &streammeta);
+        ctx = nirvana_streaming_update_start(instance_asyn, &progid, &streammeta);
+        
         if(ctx == NULL)
         {
             return PROT_STATE_DROPME;
         }
-        nirvana_streaming_asyn_update_meta(ctx, opt, opt_num);
+        nirvana_streaming_update_meta(ctx, opt, opt_num);
         qq_pme->ctx = ctx;
         newfile->ctx = ctx;
 #endif
+        MESA_handle_runtime_log(runtime_log,RLOG_LV_INFO,"match","create %d file\tuuid:%s ",qq_pme->id,qq_pme->uuid);
         MESA_htable_add(file_htable,(u_char *) qq_pme->uuid, strlen(qq_pme->uuid), (void *)newfile);
-        if (MESA_htable_get_elem_num(file_htable) > MAX_HTABLE_NUM)
-        {
-            MESA_handle_runtime_log(runtime_log,RLOG_LV_FATAL,"expire","del oldest htable");
-            MESA_htable_del_oldest_manual(file_htable, free_fileinfo, 1);
-        }
         return 0;
     }
     else
     {
         // there is a file point in htable
         // so change the pme
-        thisfile->file_len += content_len;
+        qq_pme->id = thisfile->id;
+        MESA_handle_runtime_log(runtime_log,RLOG_LV_INFO,"match","file %d existed,uuid:%s",qq_pme->id,qq_pme->uuid);
         strncpy(qq_pme->filename,thisfile->filename,strlen(thisfile->filename));
-        qq_pme->file_len = thisfile->file_len;
 #ifdef NIRVANA
         qq_pme->ctx = thisfile->ctx; 
 #endif
@@ -382,7 +387,7 @@ uchar qq_file_send_entry(stSessionInfo* session_info,void **pme,int thread_seq,s
             res = sscanf(url_decoded,"%*d.%*d.%*d.%*d:%*d/ftn_handler?bmd5=%32s",bmd5);
             if(res>0)
             {
-                MESA_handle_runtime_log(runtime_log,RLOG_LV_DEBUG,"extract","find header,bmd5=%s",bmd5);
+                //MESA_handle_runtime_log(runtime_log,RLOG_LV_DEBUG,"extract","find header,bmd5=%s",bmd5);
                 qq_pme->next_flag = 0;
                 // separator_flag is 1 means that the next TCP packet contains a separator
             }
@@ -403,21 +408,16 @@ uchar qq_file_send_entry(stSessionInfo* session_info,void **pme,int thread_seq,s
                 // looking for end suffix 
                 if(qq_pme->send_type == ONLINE_SEND)
                 {
-                    res = parse_end_suffix(content,content_len,online_feature);
+                    res = parse_end_suffix(content,content_len,online_feature,qq_pme->id);
                 }
                 else
                 {
-                    res = parse_end_suffix(content,content_len,offline_feature);
+                    res = parse_end_suffix(content,content_len,offline_feature,qq_pme->id);
                 }
                 if(res == 0)
                 {
-#ifdef NIRVANA
-                    nirvana_streaming_asyn_update_end(qq_pme->ctx);
-                    nirvana_streaming_asyn_update_cancel(qq_pme->ctx);
-#endif          
-                    MESA_handle_runtime_log(runtime_log,RLOG_LV_INFO,"finish","save file uuid=%s,size:%d",qq_pme->uuid,qq_pme->file_len);
-                    MESA_handle_runtime_log(runtime_log,RLOG_LV_INFO,"finish","total get %d file",file_cnt);
-                    MESA_htable_del(file_htable,(u_char *)qq_pme->uuid,strlen(qq_pme->uuid),free_fileinfo);
+                    MESA_handle_runtime_log(runtime_log,RLOG_LV_INFO,"finish","file %d , size: %lld",qq_pme->id, qq_pme->file_len);
+                    MESA_htable_del(file_htable,(u_char *)qq_pme->uuid,UUID_LENGTH,free_fileinfo);
                 }
             }
             else
@@ -433,11 +433,9 @@ uchar qq_file_send_entry(stSessionInfo* session_info,void **pme,int thread_seq,s
                         {
                             // If both matches fail, the rule is wrong or the traffic is wrong
                             MESA_handle_runtime_log(runtime_log,RLOG_LV_FATAL,"match","feature fail,check rule");
-                            
-#ifdef NIRVANA          
-                            nirvana_streaming_asyn_update_end(qq_pme->ctx);
-                            nirvana_streaming_asyn_update_cancel(qq_pme->ctx);
-#endif     
+                            // if the current http stream is transferring a file with a half,and the rule matches fail
+                            // there will be a streamctx still open and no way to close
+                            // but this situation seems won't happen
                             return PROT_STATE_DROPME;
                         }
                         else
@@ -449,8 +447,9 @@ uchar qq_file_send_entry(stSessionInfo* session_info,void **pme,int thread_seq,s
                     }
                     qq_pme->next_flag = 1;
                     content_len = qq_pme->content_len;
-                    match_uuid(&qq_pme,content_len);
+                    match_uuid(&qq_pme);
                 }
+                qq_pme->file_len += content_len;
                 if(locally_save == 1)
                 {
                     //save file to disk
@@ -459,7 +458,7 @@ uchar qq_file_send_entry(stSessionInfo* session_info,void **pme,int thread_seq,s
                     fclose(fp);
                 }
 #ifdef NIRVANA
-                nirvana_streaming_asyn_update_data(qq_pme->ctx, content, content_len);
+                nirvana_streaming_update_data(qq_pme->ctx, content, content_len);
 #endif      
             }
             break;
@@ -494,7 +493,7 @@ void qq_file_send_init()
     printf("qq_file_send init\n");
     int read_res = 0;
 
-    runtime_log = MESA_create_runtime_log_handle(LOG_PATH, 10);
+    runtime_log = MESA_create_runtime_log_handle(LOG_PATH, 20);
 
     const char *inf_file = "./plug/business/qq_file_send/qq_file_send.inf";
     read_res += read_profile_of_separator(inf_file,&online_feature,"ONLINE_FEATURE",ONLINE_SEND);
@@ -520,17 +519,18 @@ void qq_file_send_init()
     nirvana_client_init();
 #endif
 
-    // init a hashtable,key is uuid,value is pme
-    file_htable = MESA_htable_born();
-    int *psafe = (int *)malloc(sizeof(int));
-    int *pexpire_time = (int *)malloc(sizeof(int));
-	*psafe = 1;
-    *pexpire_time = 60;
-	MESA_htable_set_opt(file_htable,MHO_THREAD_SAFE,psafe,sizeof(int));
-    // htable will expire in 60 second
-    MESA_htable_set_opt(file_htable,MHO_EXPIRE_TIME,pexpire_time,sizeof(int));
-
-	MESA_htable_mature(file_htable);
-	MESA_htable_print_crtl(file_htable,0);
-
+    MESA_htable_create_args_t *htable_args;
+    htable_args = (MESA_htable_create_args_t *)malloc(sizeof(MESA_htable_create_args_t));
+    htable_args->thread_safe = 1;
+    htable_args->recursive = 0;
+    htable_args->hash_slot_size = 1048576;
+    htable_args->max_elem_num = MAX_HTABLE_NUM;
+    htable_args->key_comp = NULL;
+    htable_args->key2index = NULL;
+    htable_args->data_free = free_fileinfo;
+    htable_args->data_expire_with_condition = NULL;
+    htable_args->eliminate_type = 0;
+    htable_args->expire_time = 60;
+    file_htable = MESA_htable_create(htable_args,1);
+    MESA_htable_print_crtl(file_htable,0);
 }
diff --git a/qq_file_send.inf b/qq_file_send.inf
index f7ab4ff..ad5ee69 100644
--- a/qq_file_send.inf
+++ b/qq_file_send.inf
@@ -11,8 +11,7 @@ FUNC_NAME=qq_file_send_entry
 
 
 [CAPTURE]
-post_header = /ftn_handler?bmd5=
-locally_save = 1
+locally_save = 0
 save_directory = /home/jiangph
 
 
@@ -29,11 +28,9 @@ end_suffix = abcd987600000000000000000000000901
 
 [OFFLINE_FEATURE]
 separator_header = abcd9876000003ef0000
-#separator_header = [171,205,152,118,0,0,3,239,0,0]
 header_len = 10
 uuid_deviation = 83 
 uuid_suffix = 3f3f
-#uuid_suffix = [63,63]
 uuid_suffix_len = 2
 content_deviation = 364
 end_suffix_len = 17