prefilter: fix fields buffered order

4 files changed, 136 insertions, 85 deletions

diff --git a/src/osfp_common.c b/src/osfp_common.c
index 914bf1d..8ac9ac3 100644
--- a/src/osfp_common.c
+++ b/src/osfp_common.c
@@ -58,8 +58,9 @@ void osfp_profile_counter_update(struct osfp_profile_counter *profile, unsigned
 void osfp_profile_print_stats(void)
 {
     osfp_profile_counter_print(&osfp_profile_fingerprinting, "fingerprinting");
+    osfp_profile_counter_print(&osfp_profile_prefilter, "prefilter");
     osfp_profile_counter_print(&osfp_profile_score, "score");
-    osfp_profile_counter_print(&osfp_profile_result_build, "result_build");
+    osfp_profile_counter_print(&osfp_profile_result_build, "result build");
     osfp_profile_counter_print(&osfp_profile_result_export, "result export");
 }
 
diff --git a/src/osfp_fingerprint.c b/src/osfp_fingerprint.c
index 1d28109..c57ba77 100644
--- a/src/osfp_fingerprint.c
+++ b/src/osfp_fingerprint.c
@@ -65,6 +65,87 @@ struct osfp_fingerprint_field fp_fields[OSFP_FIELD_MAX] = {
     {OSFP_FINGERPRINT_FIELD_NAME_OS,                      0, OSFP_FIELD_TYPE_STRING,  0,      NULL,    0},
 };
 
+static char option_to_ascii(unsigned char type)
+{
+    switch (type) {
+        case OSFP_TCP_OPT_EOL:
+            return 'E';
+        case OSFP_TCP_OPT_NOP:
+            return 'N';
+        case OSFP_TCP_OPT_MSS:
+            return 'M';
+        case OSFP_TCP_OPT_WSCALE:
+            return 'W';
+        case OSFP_TCP_OPT_SACKOK:
+            return 'S';
+        case OSFP_TCP_OPT_SACK:
+            return 'K';
+        case OSFP_TCP_OPT_ECHO:
+            return 'J';
+        case OSFP_TCP_OPT_ECHOREPLY:
+            return 'F';
+        case OSFP_TCP_OPT_TIMESTAMP:
+            return 'T';
+        case OSFP_TCP_OPT_POCONN:
+            return 'P';
+        case OSFP_TCP_OPT_POSVC:
+            return 'R';
+        default:
+            return 'U';
+    }
+}
+
+static unsigned int compute_ip_ttl(unsigned int ip_ttl)
+{
+    if (ip_ttl >= 0 && ip_ttl <= 32) {
+        ip_ttl = 32;
+    } else if (ip_ttl > 32 && ip_ttl <= 64) {
+        ip_ttl = 64;
+    } else if (ip_ttl > 64 && ip_ttl <= 128) {
+        ip_ttl = 128;
+    } else {
+        ip_ttl = 255;
+    }
+    return ip_ttl;
+}
+
+static unsigned int decode_tcp_options(struct osfp_tcp_opt *tcp_opts, unsigned int max_opt_cnt, unsigned char *data, unsigned int len)
+{
+    unsigned int offset = 0;
+    unsigned int tcp_opt_cnt = 0;
+
+    unsigned char type;
+    unsigned char olen;
+    unsigned char *odata;
+
+    while (offset < len && tcp_opt_cnt < max_opt_cnt) {
+        type = *(data + offset);
+
+        if (type == OSFP_TCP_OPT_EOL || type == OSFP_TCP_OPT_NOP) {
+            olen = 1;
+        } else {
+            olen = *(data + offset + 1);
+            if (olen < 2) {
+                break;
+            }
+        }
+
+        if (offset + olen > len) {
+            break;
+        }
+
+        odata = (olen > 2) ? (data + offset + 2) : NULL;
+
+        tcp_opts[tcp_opt_cnt].type = type;
+        tcp_opts[tcp_opt_cnt].len = olen;
+        tcp_opts[tcp_opt_cnt].data = odata;
+
+        offset += olen;
+        tcp_opt_cnt++;
+    }
+    return tcp_opt_cnt;
+}
+
 static char *osfp_fingerprint_tcp_options_to_ordered(char *tcp_options, unsigned int len)
 {
     int i;
@@ -158,6 +239,11 @@ struct osfp_fingerprint *osfp_fingerprint_from_cjson(cJSON *root)
     }
 
     for (i = 0; i < OSFP_FIELD_OS; i++) {
+        if (0 == osfp_fingerprint_get_field_enabled(i)) {
+            OSFP_FP_INIT_FIELD(fp, i);
+            continue;
+        }
+
         field = cJSON_GetObjectItem(root, osfp_fingerprint_get_field_name(i));
         if (field == NULL) {
             OSFP_FP_INIT_FIELD(fp, i);
@@ -169,6 +255,14 @@ struct osfp_fingerprint *osfp_fingerprint_from_cjson(cJSON *root)
                 value_ptr = (void *)&field->valueint;
                 value_len = sizeof(field->valueint);
                 OSFP_FP_SET_FIELD(fp, i, value_ptr, value_len);
+                if (i == OSFP_FIELD_IP_ID ||
+                    i == OSFP_FIELD_TCP_TIMESTAMP ||
+                    i == OSFP_FIELD_TCP_TIMESTAMP_ECHO_REPLY) {
+                    *(unsigned int*)(fp->fields[i].value) = !!field->valueint;
+                }
+                if (i == OSFP_FIELD_IP_TTL) {
+                    *(unsigned int*)(fp->fields[i].value) = compute_ip_ttl((unsigned int)field->valueint);
+                }
                 break;
             case cJSON_String:
                 value_ptr = (void *)field->valuestring;
@@ -228,6 +322,11 @@ int osfp_fingerprint_from_json(struct osfp_fingerprint *fp, char *json_str)
     }
 
     for (i = 0; i < OSFP_FIELD_OS; i++) {
+        if (0 == osfp_fingerprint_get_field_enabled(i)) {
+            OSFP_FP_INIT_FIELD(fp, i);
+            continue;
+        }
+
         field = cJSON_GetObjectItem(root, osfp_fingerprint_get_field_name(i));
         if (field == NULL) {
             goto exit;
@@ -238,6 +337,14 @@ int osfp_fingerprint_from_json(struct osfp_fingerprint *fp, char *json_str)
                 value_ptr = (void *)&field->valueint;
                 value_len = sizeof(field->valueint);
                 OSFP_FP_SET_FIELD(fp, i, value_ptr, value_len);
+                if (i == OSFP_FIELD_IP_ID ||
+                    i == OSFP_FIELD_TCP_TIMESTAMP ||
+                    i == OSFP_FIELD_TCP_TIMESTAMP_ECHO_REPLY) {
+                    *(unsigned int*)(fp->fields[i].value) = !!field->valueint;
+                }
+                if (i == OSFP_FIELD_IP_TTL) {
+                    *(unsigned int*)(fp->fields[i].value) = compute_ip_ttl((unsigned int)field->valueint);
+                }
                 break;
             case cJSON_String:
                 value_ptr = (void *)field->valuestring;
@@ -260,93 +367,13 @@ exit:
     return ret;
 }
 
-static char option_to_ascii(unsigned char type)
-{
-    switch (type) {
-        case OSFP_TCP_OPT_EOL:
-            return 'E';
-        case OSFP_TCP_OPT_NOP:
-            return 'N';
-        case OSFP_TCP_OPT_MSS:
-            return 'M';
-        case OSFP_TCP_OPT_WSCALE:
-            return 'W';
-        case OSFP_TCP_OPT_SACKOK:
-            return 'S';
-        case OSFP_TCP_OPT_SACK:
-            return 'K';
-        case OSFP_TCP_OPT_ECHO:
-            return 'J';
-        case OSFP_TCP_OPT_ECHOREPLY:
-            return 'F';
-        case OSFP_TCP_OPT_TIMESTAMP:
-            return 'T';
-        case OSFP_TCP_OPT_POCONN:
-            return 'P';
-        case OSFP_TCP_OPT_POSVC:
-            return 'R';
-        default:
-            return 'U';
-    }
-}
-
-static unsigned int compute_ip_ttl(unsigned int ip_ttl)
-{
-    if (ip_ttl >= 0 && ip_ttl <= 32) {
-        ip_ttl = 32;
-    } else if (ip_ttl > 32 && ip_ttl <= 64) {
-        ip_ttl = 64;
-    } else if (ip_ttl > 64 && ip_ttl <= 128) {
-        ip_ttl = 128;
-    } else {
-        ip_ttl = 255;
-    }
-    return ip_ttl;
-}
-
-static unsigned int decode_tcp_options(struct osfp_tcp_opt *tcp_opts, unsigned int max_opt_cnt, unsigned char *data, unsigned int len)
-{
-    unsigned int offset = 0;
-    unsigned int tcp_opt_cnt = 0;
-
-    unsigned char type;
-    unsigned char olen;
-    unsigned char *odata;
-
-    while (offset < len && tcp_opt_cnt < max_opt_cnt) {
-        type = *(data + offset);
-
-        if (type == OSFP_TCP_OPT_EOL || type == OSFP_TCP_OPT_NOP) {
-            olen = 1;
-        } else {
-            olen = *(data + offset + 1);
-            if (olen < 2) {
-                break;
-            }
-        }
-
-        if (offset + olen > len) {
-            break;
-        }
-
-        odata = (olen > 2) ? (data + offset + 2) : NULL;
-
-        tcp_opts[tcp_opt_cnt].type = type;
-        tcp_opts[tcp_opt_cnt].len = olen;
-        tcp_opts[tcp_opt_cnt].data = odata;
-
-        offset += olen;
-        tcp_opt_cnt++;
-    }
-    return tcp_opt_cnt;
-}
-
 static int osfp_fingerprinting_tcp_option(unsigned char *opt_data, unsigned int opt_len, struct osfp_fingerprint *fp)
 {
     int ret,i;
 
     unsigned int tcp_mss;
     unsigned int tcp_ws;
+    unsigned int tcp_ts;
     unsigned int tcp_ter;
     unsigned int tcp_opt_cnt;
     struct osfp_tcp_opt tcp_opts[OSFP_TCP_OPTMAX];
@@ -403,7 +430,9 @@ static int osfp_fingerprinting_tcp_option(unsigned char *opt_data, unsigned int
                 if (opt->len != OSFP_TCP_OPT_TS_LEN) {
                     break;
                 }
-                tcp_ter = ntohl(*(unsigned int *)(opt->data + 4));
+                tcp_ts = !!ntohl(*(unsigned int *)(opt->data));
+                OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_TIMESTAMP, &tcp_ts, sizeof(tcp_ts));
+                tcp_ter = !!ntohl(*(unsigned int *)(opt->data + 4));
                 OSFP_FP_SET_FIELD(fp, OSFP_FIELD_TCP_TIMESTAMP_ECHO_REPLY, &tcp_ter, sizeof(tcp_ter));
                 ret = snprintf(options + offset, sizeof(options), "%u", tcp_ter);
                 if (ret < 0 || offset + ret > maxoffset) {
diff --git a/src/osfp_score_db.c b/src/osfp_score_db.c
index 6267163..88a3bea 100644
--- a/src/osfp_score_db.c
+++ b/src/osfp_score_db.c
@@ -269,6 +269,9 @@ static int osfp_score_db_load_entry(struct osfp_score_db *score_db, cJSON *entry
         element->fp = fp;
         element->os_class = os_class;
         HASH_ADD_KEYPTR(hh, score_db->prefilter_head, fp->value_buffer, fp->value_buffer_used, element);
+    } else {
+        // TODO: same fingerprints with different os should not insert into prefilter hash table, now just tag
+        element->repeated++;
     }
 
     // field score db
@@ -414,18 +417,34 @@ exit:
 
 int osfp_score_db_prefilter(struct osfp_score_db *score_db, struct osfp_fingerprint *fp, struct osfp_os_class_score *result_score)
 {
-    int ret;
+    int ret, i;
+    unsigned int value_buffer_used = 0;
+    char value_buffer[OSFP_FINGERPRINT_VALUE_BUFFER_MAX];
     struct osfp_prefilter_hash_element *element = NULL;
 
     if (score_db->prefilter_head == NULL) {
         return 0;
     }
 
-    HASH_FIND(hh, score_db->prefilter_head, fp->value_buffer, fp->value_buffer_used, element);
+    for (i = 0; i < OSFP_FIELD_OS; i++) {
+        if (0 == osfp_fingerprint_get_field_enabled(i)) {
+            continue;
+        }
+        if (fp->fields[i].value && fp->fields[i].value_len != 0) {
+            memcpy(value_buffer + value_buffer_used, fp->fields[i].value, fp->fields[i].value_len);
+            value_buffer_used += fp->fields[i].value_len;
+        }
+    }
+
+    HASH_FIND(hh, score_db->prefilter_head, value_buffer, value_buffer_used, element);
     if (element == NULL) {
         return 0;
     }
 
+    if (element->repeated) {
+        return 0;
+    }
+
     memset(result_score, 0, sizeof(struct osfp_os_class_score));
     result_score->scores[element->os_class] = OSFP_PERCENTILE;
 
diff --git a/src/osfp_score_db.h b/src/osfp_score_db.h
index b73427d..cda21f0 100644
--- a/src/osfp_score_db.h
+++ b/src/osfp_score_db.h
@@ -26,6 +26,8 @@ struct osfp_prefilter_hash_element {
     struct osfp_fingerprint *fp;
     char *fp_json;
     unsigned int os_class;
+
+    unsigned int repeated;
     UT_hash_handle hh;
 };