performance: matric serialize

6 files changed, 114 insertions, 137 deletions

diff --git a/src/metrics/metric.c b/src/metrics/metric.c
index d66fafe..12ff1a7 100644
--- a/src/metrics/metric.c
+++ b/src/metrics/metric.c
@@ -5,9 +5,9 @@
 #include <string.h>
 #include <stdarg.h>
 
-#include "mpack/mpack.h"
 #include "uthash.h"
 
+#include "serializer.h"
 #include "metric.h"
 #include "fieldstat.h"
 #include "tags/cell_manager.h"
@@ -137,23 +137,17 @@ static void metric_scheme_counter_free(struct metric_measure_data *data)
         "value": 0
         "mode": 0(sum) / 1(max)  /  2(min)
     }
+value: long long | mode: uint32
 */
 static void metric_scheme_counter_serialize(const struct metric_measure_data *data, char **blob, size_t *blob_size)
 {
     const struct metric_counter_or_gauge *counter = data->counter;
-    mpack_writer_t writer;
-    mpack_writer_init_growable(&writer, blob, blob_size);
-    mpack_build_map(&writer);
+    struct fs_reader *reader = fs_reader_new();
 
-    mpack_write_cstr(&writer, "value");
-    mpack_write_i64(&writer, counter->value);
-    mpack_write_cstr(&writer, "mode");
-    mpack_write_i8(&writer, counter->mode);
+    fs_reader_read_longlong(reader, counter->value);
+    fs_reader_read_uint(reader, counter->mode);
 
-    mpack_complete_map(&writer);
-    if (mpack_writer_destroy(&writer) != mpack_ok) {
-        assert(0);
-    }
+    fs_reader_finalize(reader, blob, blob_size);
 }
 
 static int metric_scheme_counter_merge(struct metric_measure_data *pthis, const struct metric_measure_data *from)
@@ -193,21 +187,15 @@ static int metric_scheme_counter_copy(struct metric_measure_data *pthis, const s
 
 struct metric_measure_data *metric_scheme_counter_deserialize(const char *blob, size_t blob_size) 
 {
-    mpack_tree_t content_tree;
-    mpack_tree_init_data(&content_tree, blob, blob_size);
-    mpack_tree_parse(&content_tree);
-    mpack_node_t content_root = mpack_tree_root(&content_tree);
-
+    struct fs_writer *writer = fs_writer_new(blob, blob_size);
     struct metric_measure_data *ret = (struct metric_measure_data *)malloc(sizeof(struct metric_measure_data));
     struct metric_counter_or_gauge *counter = (struct metric_counter_or_gauge *)malloc(sizeof(struct metric_counter_or_gauge));
     ret->counter = counter;
 
-    counter->value = mpack_node_i64(mpack_node_map_cstr(content_root, "value"));
-    counter->mode = mpack_node_i8(mpack_node_map_cstr(content_root, "mode"));
+    fs_writer_write_longlong(writer, &counter->value);
+    fs_writer_write_uint(writer, (uint32_t *)&counter->mode);
 
-    if (mpack_tree_destroy(&content_tree) != mpack_ok) {
-        assert(0);
-    }
+    fs_writer_free(writer);
     return ret;
 }
 
@@ -471,75 +459,73 @@ struct metric *metric_copy(const struct metric *src)
     "min": <lowest_trackable_value>
     "max": <highest_trackable_value>
     "prec": <significant_figures>
+
+type:uint32 | min: long long | max: long long | prec: uint32
 }
 */
 void metric_serialize_parameters(const struct metric *metric, char **blob, size_t *blob_size)
 {
     struct metric_parameter *para = metric->info->paras;
-    mpack_writer_t writer;
-    mpack_writer_init_growable(&writer, blob, blob_size);
-    mpack_build_map(&writer);
+    struct fs_reader *reader = fs_reader_new();
 
-    mpack_write_cstr(&writer, "type");
-    mpack_write_i32(&writer, metric->info->type);
+    fs_reader_read_uint(reader, metric->info->type);
 
     switch (metric->info->type)
     {
     case METRIC_TYPE_COUNTER:
-        mpack_write_cstr(&writer, "merge_type");
-        mpack_write_i32(&writer, para->counter.mode);
+        fs_reader_read_uint(reader, para->counter.mode);
     case METRIC_TYPE_HLL:
-        mpack_write_cstr(&writer, "prec");
-        mpack_write_i32(&writer, para->hll.precision);
+        fs_reader_read_uint(reader, para->hll.precision);
         break;
     case METRIC_TYPE_HISTOGRAM:
-        mpack_write_cstr(&writer, "min");
-        mpack_write_i64(&writer, para->hdr.lowest_trackable_value);
-        mpack_write_cstr(&writer, "max");
-        mpack_write_i64(&writer, para->hdr.highest_trackable_value);
-        mpack_write_cstr(&writer, "prec");
-        mpack_write_i8(&writer, para->hdr.significant_figures);
+        fs_reader_read_longlong(reader, para->hdr.lowest_trackable_value);
+        fs_reader_read_longlong(reader, para->hdr.highest_trackable_value);
+        fs_reader_read_uint(reader, para->hdr.significant_figures);
         break;
     default:
         break;
     }
-    mpack_complete_map(&writer);
 
-    mpack_writer_destroy(&writer);
+    fs_reader_finalize(reader, blob, blob_size);
 }
 
 struct metric_parameter *deserialize_parameters(const char *blob, size_t blob_size)
 {
-    mpack_tree_t tree;
-    mpack_tree_init_data(&tree, blob, blob_size);
-    mpack_tree_parse(&tree);
-    mpack_node_t root = mpack_tree_root(&tree);
+    struct fs_writer *writer = fs_writer_new(blob, blob_size);
+
+    enum metric_type type;
+    fs_writer_write_uint(writer, (uint32_t *)&type);
 
-    mpack_node_t type_obj = mpack_node_map_cstr(root, "type");
-    enum metric_type type = mpack_node_i32(type_obj);
     struct metric_parameter *field_para = NULL;
     switch (type) {
         case METRIC_TYPE_COUNTER:{
-            int counter_type = mpack_node_i32(mpack_node_map_cstr(root, "merge_type"));
+            int counter_type;
+            fs_writer_write_uint(writer, (uint32_t *)&counter_type);
             field_para = construct_parameters(type, counter_type);
         }
         break;
         case METRIC_TYPE_HLL: {
-            int precision = mpack_node_i32(mpack_node_map_cstr(root, "prec"));
+            int precision;
+            fs_writer_write_uint(writer, (uint32_t *)&precision);
             field_para = construct_parameters(type, precision);
         }
         break;
         case METRIC_TYPE_HISTOGRAM: {
-            int64_t min = mpack_node_i64(mpack_node_map_cstr(root, "min"));
-            int64_t max = mpack_node_i64(mpack_node_map_cstr(root, "max"));
-            int8_t prec = mpack_node_i8(mpack_node_map_cstr(root, "prec"));
+            long long min;
+            long long max;
+            int8_t prec;
+            fs_writer_write_longlong(writer, &min);
+            fs_writer_write_longlong(writer, &max);
+            fs_writer_write_uint(writer, (uint32_t *)&prec);
+
             field_para = construct_parameters(type, min, max, prec);
         }
         break;
         default:
             break;
     }
-    mpack_tree_destroy(&tree);
+    
+    fs_writer_free(writer);
     return field_para;
 }
 
@@ -547,41 +533,31 @@ struct metric_parameter *deserialize_parameters(const char *blob, size_t blob_si
 {
     "id":<cell_id>,
     "PL":<content_blob>
+
+id:uint | PL:bin
 }
 */
-void mpack_blob_wrap_basic_info(int cell_id, const char *blob, size_t blob_size, char **new_blob, size_t *new_blob_size)
+void blob_wrap_basic_info(int cell_id, const char *blob, size_t blob_size, char **new_blob, size_t *new_blob_size)
 {
-    mpack_writer_t writer;
-    mpack_writer_init_growable(&writer, new_blob, new_blob_size);
-    mpack_start_map(&writer, 2);
-    mpack_write_cstr(&writer, "id");
-    mpack_write_int(&writer, cell_id);
-    mpack_write_cstr(&writer, "PL");
-    mpack_write_bin(&writer, blob, blob_size);
-    mpack_finish_map(&writer);
-    mpack_writer_destroy(&writer);
+    struct fs_reader *reader = fs_reader_new();
+
+    fs_reader_read_uint(reader, cell_id);
+    fs_reader_read_bin(reader, blob, blob_size);
+
+    fs_reader_finalize(reader, new_blob, new_blob_size);
 }
 
 // user must free new_blob
-void mpack_blob_unpack_basic_info(const char *blob, size_t blob_size, char **new_blob, size_t *new_blob_size, int *cell_id)
+void blob_unpack_basic_info(const char *blob, size_t blob_size, char **new_blob, size_t *new_blob_size, int *cell_id)
 {
-    mpack_tree_t tree;
-    mpack_tree_init_data(&tree, blob, blob_size);
-    mpack_tree_parse(&tree);
-    mpack_node_t root = mpack_tree_root(&tree);
-
-    mpack_node_t id_node = mpack_node_map_cstr(root, "id");
-    *cell_id = mpack_node_int(id_node);
+    struct fs_writer *writer = fs_writer_new(blob, blob_size);
 
-    mpack_node_t payload_node = mpack_node_map_cstr(root, "PL");
-    *new_blob_size = mpack_node_bin_size(payload_node);
-    *new_blob = (char *)malloc(*new_blob_size);
-    memcpy(*new_blob, mpack_node_bin_data(payload_node), *new_blob_size);
+    fs_writer_write_uint(writer, (uint32_t *)cell_id);
+    fs_writer_write_bin(writer, new_blob, new_blob_size);
 
-    mpack_tree_destroy(&tree);
+    fs_writer_free(writer);
 }
 
-
 /*
 metric blob:
     {
@@ -593,33 +569,30 @@ metric blob:
             "id":<cell_id>
             "PL":<content_blob(gen by scheme)>
         }]
-}*/
+}
+
+name:string | type:uint | para:bin | data_id:int_array | data_payload:bin_array
+*/
 int metric_serialize(const struct metric *pthis, char **blob, size_t *blob_size)
 {
-    char *para_blob = NULL;
-    size_t para_blob_size = 0;
-    mpack_writer_t writer;
-    mpack_writer_init_growable(&writer, blob, blob_size);
+    struct fs_reader *reader = fs_reader_new();
 
-    mpack_build_map(&writer);
-    mpack_write_cstr(&writer, "name");
-    mpack_write_str(&writer, pthis->info->name, strlen(pthis->info->name));
-    mpack_write_cstr(&writer, "type");
-    mpack_write_i32(&writer, pthis->info->type);
+    fs_reader_read_str(reader, pthis->info->name);
+    fs_reader_read_uint(reader, pthis->info->type);
 
-    mpack_write_cstr(&writer, "para");
+    char *para_blob = NULL;
+    size_t para_blob_size = 0;
     metric_serialize_parameters(pthis, &para_blob, &para_blob_size);
-    mpack_write_bin(&writer, para_blob, para_blob_size);
+    fs_reader_read_bin(reader, para_blob, para_blob_size);
     free(para_blob);
 
-    mpack_write_cstr(&writer, "data");
     int *cell_ids = NULL;
     size_t cell_ids_size = 0;
     metric_get_cell_ids(pthis, &cell_ids, &cell_ids_size);
     if (cell_ids == NULL) {
-        mpack_write_nil(&writer);
+        fs_reader_read_nil(reader);
     } else {
-        mpack_build_array(&writer);
+        fs_reader_start_bin_array(reader, cell_ids_size);
         for (int i = 0; i < cell_ids_size; i++) {
             struct metric_measure_data *data = metric_find_one_cell(pthis, cell_ids[i]);
 
@@ -629,54 +602,66 @@ int metric_serialize(const struct metric *pthis, char **blob, size_t *blob_size)
 
             char *tmp_data_wrapped_blob = NULL;
             size_t tmp_data_wrapped_blob_size = 0;
-            mpack_blob_wrap_basic_info(cell_ids[i], data_blob, data_blob_size, &tmp_data_wrapped_blob, &tmp_data_wrapped_blob_size);
-            mpack_write_bin(&writer, tmp_data_wrapped_blob, tmp_data_wrapped_blob_size);
+            blob_wrap_basic_info(cell_ids[i], data_blob, data_blob_size, &tmp_data_wrapped_blob, &tmp_data_wrapped_blob_size);
+            fs_reader_read_bin(reader, tmp_data_wrapped_blob, tmp_data_wrapped_blob_size);
             free(data_blob);
             free(tmp_data_wrapped_blob);
         }
+
         free(cell_ids);
-        mpack_complete_array(&writer);
     }
 
-    mpack_complete_map(&writer);
-    if (mpack_writer_destroy(&writer) != mpack_ok) {
-        return -1;
-    }
+    fs_reader_finalize(reader, blob, blob_size);
 
     return 0;
 }
 
 struct metric *metric_deserialize(const char *blob, size_t blob_size)
 {
-    mpack_tree_t tree;
-    mpack_tree_init_data(&tree, blob, blob_size);
-    mpack_tree_parse(&tree);
-    mpack_node_t root = mpack_tree_root(&tree);
+    struct fs_writer *writer = fs_writer_new(blob, blob_size);
 
-    char *metric_name = mpack_node_cstr_alloc(mpack_node_map_cstr(root, "name"), 0xFFFF); // 0xFFFF: a very large number
-    enum metric_type type = mpack_node_i32(mpack_node_map_cstr(root, "type"));
-    mpack_node_t para_obj = mpack_node_map_cstr(root, "para");
-    struct metric_parameter *metric_para = deserialize_parameters(mpack_node_bin_data(para_obj), mpack_node_bin_size(para_obj));
+    char *metric_name;
+    fs_writer_write_str(writer, &metric_name);
+    enum metric_type type;
+    fs_writer_write_uint(writer, (uint32_t *)&type);
+
+    char *para_blob;
+    size_t para_blob_size;
+    fs_writer_write_bin(writer, &para_blob, &para_blob_size);
+    struct metric_parameter *metric_para = deserialize_parameters(para_blob, para_blob_size);
     struct metric *metric_out = metric_new(metric_name, type, metric_para);
     free(metric_name);
+    free(para_blob);
 
-    mpack_node_t data_arr = mpack_node_map_cstr(root, "data");
-    if (mpack_node_is_nil(data_arr)) {
-        mpack_tree_destroy(&tree);
+    int ret = fs_writer_expect_nil(writer);
+    assert(ret >= 0);
+    if (ret == 1) { // no cells in this metric
+        fs_writer_free(writer);
         return metric_out;
     }
     
-    for (int i = 0; i < mpack_node_array_length(data_arr); i++) {
-        mpack_node_t data_node = mpack_node_array_at(data_arr, i);
+    char **data_blob_raw = NULL;
+    size_t *data_blob_raw_size = NULL;
+    size_t n_data_blob_raw = 0;
+    fs_writer_write_bin_array(writer, &data_blob_raw, &n_data_blob_raw, &data_blob_raw_size);
+
+    for (int i = 0; i < n_data_blob_raw; i++) {
         int cell_id;
         char *data_blob = NULL;
         size_t data_blob_size = 0;
-        mpack_blob_unpack_basic_info(mpack_node_bin_data(data_node), mpack_node_bin_size(data_node), &data_blob, &data_blob_size, &cell_id);
+        blob_unpack_basic_info(data_blob_raw[i], data_blob_raw_size[i], &data_blob, &data_blob_size, &cell_id);
         struct metric_measure_data *data = metric_out->scheme->deserialize(data_blob, data_blob_size);
         free(data_blob);
         metric_add_one_cell(metric_out, cell_id, data);
     }
-    mpack_tree_destroy(&tree);
+    
+    for (int i = 0; i < n_data_blob_raw; i++) {
+        free(data_blob_raw[i]);
+    }
+    free(data_blob_raw);
+    free(data_blob_raw_size);
+    fs_writer_free(writer);
+
     return metric_out;
 }
 
diff --git a/src/tags/cell_manager.c b/src/tags/cell_manager.c
index 99a89db..a10c5e2 100644
--- a/src/tags/cell_manager.c
+++ b/src/tags/cell_manager.c
@@ -3,7 +3,6 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "mpack/mpack.h"
 #include "fieldstat.h"
 #include "tags/cell_manager.h"
 #include "tags/heavy_keeper.h"
@@ -214,12 +213,13 @@ int cell_manager_serialize(const struct cell_manager *pthis, char **blob, size_t
     fs_reader_read_uint(reader, (uint32_t)pthis->sampling_mode);
     fs_reader_read_uint(reader, (uint32_t)pthis->max_cell_num);
 
+
     if (pthis->sampling_mode == SAMPLING_MODE_COMPREHENSIVE) {
-        // mpack_write_cstr(&writer, "id_tag_array");
-        // mpack_build_array(&writer);
-        int id_rec[pthis->max_cell_id];
+        int n_cell = pthis->max_cell_id + 1;
+
+        int id_rec[n_cell];
         int actual_len = 0;
-        for (int i = 0; i <= pthis->max_cell_id; i++) {
+        for (int i = 0; i < n_cell; i++) {
             if (pthis->id_tag_array[i] == NULL) {
                 continue;
             }
@@ -247,7 +247,6 @@ int cell_manager_serialize(const struct cell_manager *pthis, char **blob, size_t
 
     fs_reader_finalize(reader, blob, blob_size);
 
-    fs_reader_free(reader);
     return 0;
 }
 
diff --git a/src/tags/my_ut_hash.c b/src/tags/my_ut_hash.c
index 6e9bff1..6f6a170 100644
--- a/src/tags/my_ut_hash.c
+++ b/src/tags/my_ut_hash.c
@@ -199,8 +199,6 @@ void fieldtag_serialize(const struct fieldstat_tag *tag_list, char **blob, size_
             break;
     }
     fs_reader_finalize(reader, blob, blob_size);
-
-    fs_reader_free(reader);
 }
 
 void fieldtag_deserialize(const char *blob, size_t blob_size, struct fieldstat_tag *tag)
@@ -247,8 +245,6 @@ void fieldtag_list_serialize(const struct fieldstat_tag *tag_list, size_t tag_nu
     }
 
     fs_reader_finalize(reader, blob, blob_size);
-
-    fs_reader_free(reader);
 }
 
 void fieldtag_list_deserialize(const char *blob, size_t blob_size, struct fieldstat_tag **tag_list_out, size_t *tag_num_out)
diff --git a/src/utils/serializer.c b/src/utils/serializer.c
index 124f9d1..9b86eb7 100644
--- a/src/utils/serializer.c
+++ b/src/utils/serializer.c
@@ -144,11 +144,6 @@ struct fs_reader *fs_reader_new() {
     return reader;
 }
 
-void fs_reader_free(struct fs_reader *reader) {
-    free(reader->data);
-    free(reader);
-}
-
 const char *fs_reader_unwrap(const struct fs_reader *reader, size_t *size) {
     *size = reader->cursor;
     return reader->data;
@@ -157,9 +152,7 @@ const char *fs_reader_unwrap(const struct fs_reader *reader, size_t *size) {
 void fs_reader_finalize(struct fs_reader *reader, char **blob, size_t *size) {
     *size = reader->cursor;
     *blob = reader->data;
-    reader->data = NULL;
-    reader->cursor = 0;
-    reader->data_buffer_size = 0;
+    free(reader);
 }
 
 /* -------------------------------------------------------------------------- */
diff --git a/src/utils/serializer.h b/src/utils/serializer.h
index fb85151..d2e7e80 100644
--- a/src/utils/serializer.h
+++ b/src/utils/serializer.h
@@ -10,7 +10,8 @@ extern "C" {
 struct fs_reader;
 
 struct fs_reader *fs_reader_new();
-void fs_reader_free(struct fs_reader *reader);
+void fs_reader_finalize(struct fs_reader *reader, char **blob, size_t *size);
+const char *fs_reader_unwrap(const struct fs_reader *reader, size_t *size);
 
 void fs_reader_read_uint(struct fs_reader *reader, uint32_t value);
 void fs_reader_read_longlong(struct fs_reader *reader, uint64_t value);
@@ -21,8 +22,7 @@ void fs_reader_read_str(struct fs_reader *reader, const char *value);
 void fs_reader_start_bin_array(struct fs_reader *reader, size_t size);
 void fs_reader_read_int_array(struct fs_reader *reader, const int *value, size_t size); 
 
-const char *fs_reader_unwrap(const struct fs_reader *reader, size_t *size);
-void fs_reader_finalize(struct fs_reader *reader, char **blob, size_t *size);
+
 /* ------------------------------------------------------------------------ */
 struct fs_writer;
 struct fs_writer *fs_writer_new(const char *blob, size_t size);
diff --git a/test/test_performance.cpp b/test/test_performance.cpp
index 9f654ff..e311b66 100644
--- a/test/test_performance.cpp
+++ b/test/test_performance.cpp
@@ -389,7 +389,7 @@
 clock_t perform_serialize_test(std::function<void (struct fieldstat*, int, int, int)> metric_add_func, 
     std::function<int(struct fieldstat*, int)> metric_register_func, enum sampling_mode mode)
 {
-    const int MAX_CELL_NUM = 1000000;
+    const int MAX_CELL_NUM = 10000;
     Fieldstat_tag_list_wrapper *tags[MAX_CELL_NUM];
     for (int i = 0; i < MAX_CELL_NUM; i++) {
         tags[i] = new Fieldstat_tag_list_wrapper("my key", i);
@@ -404,17 +404,21 @@ clock_t perform_serialize_test(std::function<void (struct fieldstat*, int, int,
 
     char *blob;
     size_t blob_size;
+    printf("start\n");
+
     clock_t start = clock();
-    printf("getchar\n");
-    getchar();
+    // printf("getchar\n");
+    // getchar();
     fieldstat_serialize(instance, &blob, &blob_size);
-    exit(0);
+    // exit(0);
     clock_t end = clock();
+    printf("end\n");
 
     fieldstat_free(instance);
     for (int i = 0; i < MAX_CELL_NUM; i++) {
         delete tags[i];
     }
+    free(blob);
 
     return end - start;
 }