histogram batch

8 files changed, 192 insertions, 49 deletions

diff --git a/include/fieldstat/fieldstat.h b/include/fieldstat/fieldstat.h
index 8268436..0e3bce9 100644
--- a/include/fieldstat/fieldstat.h
+++ b/include/fieldstat/fieldstat.h
@@ -57,13 +57,7 @@ struct field {
 	};
 };
 
-struct field_t_0_fs *pst = (fields[]);
-{
-    uint64_t hashv = 0;
-    char *serialized_dimension;
-    int bucket_in_uthash;
-    struct cell *cell;
-}
+struct histogram; 
 
 struct fieldstat;
 struct fieldstat *fieldstat_new();
@@ -160,15 +154,28 @@ int fieldstat_hll_add_fields(struct fieldstat *instance, int cube_id, int metric
     * @return Error codes.
 */
 int fieldstat_histogram_record(struct fieldstat *instance, int cube_id, int metric_id, const struct field *cell_dimensions, size_t n_dimensions, long long value);
-struct histogram; 
-struct histogram *fieldstat_histogram_fork(struct fieldstat *instance, int metric_id);
-void 
-void histogram_free(struct fieldstat_histogram *histogram);
-void histogram_reset(struct fieldstat_histogram *histogram);
-int fieldstat_histogram_merge(struct fieldstat *instance, int cube_id, int metric_id, const struct field *cell_dimensions, size_t n_dimensions, const struct histogram *src);
 
-
-struct 
+/*
+struct histogram provides a way to update the histogram metric in batch created and updated by user.
+
+  Example:
+    struct histogram *histogram = fieldstat_histogram_fork(instance, cube_id, metric_id);
+    for (int i = 0; i < many_packet_length; i++) {
+        histogram_record(histogram, packet_length[i]);
+    }
+
+    fieldstat_histogram_merge(instance, cube_id, metric_id, cell_dimensions, n_dimensions, histogram);
+    histogram_reset(histogram);
+    
+    ...
+    // after many times of histogram_record
+    histogram_free(histogram);
+*/
+struct histogram *fieldstat_histogram_fork(const struct fieldstat *instance, int cube_id, int metric_id);
+void histogram_free(struct histogram *histogram);
+void histogram_reset(struct histogram *histogram);
+int fieldstat_histogram_merge(struct fieldstat *instance, int cube_id, int metric_id, const struct field *cell_dimensions, size_t n_dimensions, const struct histogram *src);
+int histogram_record(struct histogram *histogram, long long value);
 
 /*
     * @brief Delete all the cells, also the content of every metrics. The cube and metrics are not deleted.
diff --git a/src/cube.c b/src/cube.c
index 1c3e6df..1779a0c 100644
--- a/src/cube.c
+++ b/src/cube.c
@@ -803,6 +803,29 @@ int cube_histogram_record(struct cube *cube, int metric_id, const struct field *
     return FS_OK;
 }
 
+int cube_histogram_merge(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, const struct hdr_histogram *src) {
+    if (cube->primary_metric_id == -1) {
+        return FS_ERR_CUBE_SAMPLING_NOT_INITIALIZED;
+    }
+    assert(cube->sampling_mode == SAMPLING_MODE_COMPREHENSIVE || (cube->primary_metric_id != metric_id));
+
+    const struct metric_manifest *manifest = metric_manifest_manager_get_by_id(cube->manifest_manager, metric_id);
+    if (manifest == NULL || manifest->type != METRIC_TYPE_HISTOGRAM) {
+        printf("invalid metric id\n");
+        return FS_ERR_INVALID_METRIC_ID;
+    }
+
+    struct cell *cell_data = get_cell_in_cube_generic(cube, dimensions, n_dimensions);
+    if (cell_data == NULL) {
+        printf("too many cells\n");
+        return FS_ERR_TOO_MANY_CELLS;
+    }
+
+    struct metric *metric = add_or_find_metric_in_cell(manifest, cell_data);
+    metric_histogram_merge(metric, src);
+    return FS_OK;
+}
+
 int cube_hll_add(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, const char *key, size_t key_len) {
     if (cube->primary_metric_id == -1) {
         return FS_ERR_CUBE_SAMPLING_NOT_INITIALIZED;
@@ -1388,30 +1411,10 @@ struct field_list *cube_get_identifier(const struct cube *cube) {
     return ret;
 }
 
-const char *cube_get_metric_name(const struct cube *cube, int metric_id) {
-    const struct metric_manifest *metric = metric_manifest_manager_get_by_id(cube->manifest_manager, metric_id);
-    if (metric == NULL) {
-        return NULL;
-    }
-
-    return metric->name;
-}
-
-enum metric_type cube_get_metric_type(const struct cube *cube, int metric_id) {
-    const struct metric_manifest *metric = metric_manifest_manager_get_by_id(cube->manifest_manager, metric_id);
-    if (metric == NULL) {
-        return (enum metric_type)(-1);
-    }
-
-    return metric->type;
+const struct metric_manifest *cube_get_metric_manifest_by_id(const struct cube *cube, int metric_id) {
+    return metric_manifest_manager_get_by_id(cube->manifest_manager, metric_id);
 }
 
-int cube_get_metric_id_by_name(const struct cube *cube, const char *metric_name) 
-{
-    const struct metric_manifest *metric = metric_manifest_manager_get_by_name(cube->manifest_manager, metric_name);
-    if (metric == NULL) {
-        return FS_ERR_INVALID_METRIC_NAME;
-    }
-
-    return metric->id;
+const struct metric_manifest *cube_get_metric_manifest_by_name(const struct cube *cube, const char *metric_name) {
+    return metric_manifest_manager_get_by_name(cube->manifest_manager, metric_name);
 }
diff --git a/src/cube.h b/src/cube.h
index 6710e34..973e839 100644
--- a/src/cube.h
+++ b/src/cube.h
@@ -7,6 +7,7 @@ extern "C"
 
 #include <stddef.h>
 #include <stdbool.h>
+#include "hdr/hdr_histogram.h"
 #include "fieldstat.h" // for dimensions
 #include "metric_manifest.h"
 
@@ -26,6 +27,7 @@ int cube_register_hll(struct cube *cube,const char *metric_name, unsigned char p
 int cube_register_hist(struct cube *cube,const char *metric_name, long long lowest_trackable_value, long long highest_trackable_value, int significant_figures);
 
 int cube_histogram_record(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, long long value);
+int cube_histogram_merge(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, const struct hdr_histogram *src);
 int cube_hll_add(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, const char *key, size_t key_len);
 int cube_hll_add_field(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, const struct field *item_fields, size_t n_item);
 int cube_counter_incrby(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, long long increment);
@@ -43,9 +45,9 @@ enum sampling_mode cube_get_sampling_mode(const struct cube *cube);
 void cube_get_cells(const struct cube *cube, struct field_list **cell_dimensions, size_t *n_cell);
 void cube_get_metrics_in_cell(const struct cube *cube, const struct field_list *dimensions, int **metric_id_out, size_t *n_metric_out);
 struct field_list *cube_get_identifier(const struct cube *cube);
-const char *cube_get_metric_name(const struct cube *cube, int metric_id);
-enum metric_type cube_get_metric_type(const struct cube *cube, int metric_id);
-int cube_get_metric_id_by_name(const struct cube *cube, const char *metric_name);
+const struct metric_manifest *cube_get_metric_manifest_by_id(const struct cube *cube, int metric_id);
+const struct metric_manifest *cube_get_metric_manifest_by_name(const struct cube *cube, const char *metric_name);
+
 
 /* -------------------------------------------------------------------------- */
 /*                                cube manager                                */
diff --git a/src/fieldstat.c b/src/fieldstat.c
index 86aa0b2..1becadd 100644
--- a/src/fieldstat.c
+++ b/src/fieldstat.c
@@ -6,15 +6,21 @@
 
 #include "cjson/cJSON.h"
 #include "uthash.h"
+#include "hdr/hdr_histogram.h"
 
 #include "fieldstat.h"
 #include "metrics/metric.h"
 #include "cube.h"
+#include "metric_manifest.h"
 
 struct fieldstat {
     struct cube_manager *cube_manager;
 };
 
+struct histogram {
+    struct hdr_histogram *histogram;
+};
+
 
 /* -------------------------------------------------------------------------- */
 /*                                  fieldstat                                 */
@@ -221,6 +227,65 @@ void fieldstat_calibrate(const struct fieldstat *master, struct fieldstat *repli
 }
 
 /* -------------------------------------------------------------------------- */
+/*                      histogram fast batch transaction                      */
+/* -------------------------------------------------------------------------- */
+struct histogram *fieldstat_histogram_fork(const struct fieldstat *instance, int cube_id, int metric_id) {
+    const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id);
+    if (cube == NULL) {
+        return NULL;
+    }
+
+    const struct metric_manifest *manifest = cube_get_metric_manifest_by_id(cube, metric_id);
+    if (manifest == NULL || manifest->type != METRIC_TYPE_HISTOGRAM) {
+        return NULL;
+    }
+    const struct histogram_parameters *parameters = &(manifest->parameters->hdr);
+
+    struct hdr_histogram* inner = NULL;
+    int ret = hdr_init(parameters->lowest_trackable_value, parameters->highest_trackable_value, parameters->significant_figures, &inner);
+	if (ret != 0) {
+        return NULL;
+    }
+
+    struct histogram *histogram = calloc(1, sizeof(struct histogram));
+    histogram->histogram = inner;
+
+    return histogram;
+}
+
+void histogram_free(struct histogram *histogram)
+{
+    if (histogram == NULL) {
+        return;
+    }
+    hdr_close(histogram->histogram);
+    free(histogram);
+}
+
+void histogram_reset(struct histogram *histogram) {
+    hdr_reset(histogram->histogram);
+}
+
+// cppcheck-suppress [constParameterPointer, unmatchedSuppression]
+int fieldstat_histogram_merge(struct fieldstat *instance, int cube_id, int metric_id, const struct field *cell_dimensions, size_t n_dimensions, const struct histogram *src)
+{
+    struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id);
+    if (cube == NULL) {
+        return FS_ERR_INVALID_CUBE_ID;
+    }
+
+    return cube_histogram_merge(cube, metric_id, cell_dimensions, n_dimensions, src->histogram);
+}
+
+int histogram_record(struct histogram *histogram, long long value) {
+    bool ret = hdr_record_value(histogram->histogram, value);
+    if (!ret) {
+        return FS_ERR_INVALID_PARAM;
+    }
+    return FS_OK;
+}
+
+/* -------------------------------------------------------------------------- */
 /*                                    query                                   */
 /* -------------------------------------------------------------------------- */
 void fieldstat_get_cubes(const struct fieldstat *instance, int **cube_ids, int *n_cube)
@@ -327,7 +392,11 @@ const char *fieldstat_metric_get_name(const struct fieldstat *instance, int cube
     if (cube == NULL) {
         return NULL;
     }
-    return cube_get_metric_name(cube, metric_id);
+    const struct metric_manifest *manifest = cube_get_metric_manifest_by_id(cube, metric_id);
+    if (manifest == NULL) {
+        return NULL;
+    }
+    return manifest->name;
 }
 
 enum metric_type fieldstat_metric_get_type(const struct fieldstat *instance, int cube_id, int metric_id)
@@ -336,7 +405,13 @@ enum metric_type fieldstat_metric_get_type(const struct fieldstat *instance, int
     if (cube == NULL) {
         return (enum metric_type)(-1);
     }
-    return cube_get_metric_type(cube, metric_id);
+    
+    const struct metric_manifest *manifest = cube_get_metric_manifest_by_id(cube, metric_id);
+    if (manifest == NULL) {
+        return (enum metric_type)(-1);
+    }
+
+    return manifest->type;
 }
 
 void fieldstat_cube_get_cells(const struct fieldstat *instance, int cube_id, struct field_list **cell_dimensions, size_t *n_cell)
@@ -390,5 +465,10 @@ int fieldstat_cube_get_metric_id_by_name(const struct fieldstat *instance, int c
         return FS_ERR_INVALID_CUBE_ID;
     }
 
-    return cube_get_metric_id_by_name(cube, metric_name);
+    const struct metric_manifest *manifest = cube_get_metric_manifest_by_name(cube, metric_name);
+    if (manifest == NULL) {
+        return FS_ERR_INVALID_METRIC_NAME;
+    }
+
+    return manifest->id;
 }
\ No newline at end of file
diff --git a/src/metrics/metric.c b/src/metrics/metric.c
index 95dc646..f12e6b8 100644
--- a/src/metrics/metric.c
+++ b/src/metrics/metric.c
@@ -368,6 +368,13 @@ int metric_histogram_record(struct metric *pthis, long long value) {
     return 0;
 }
 
+void metric_histogram_merge(struct metric *dest, const struct hdr_histogram *src) {
+    assert(dest->type == METRIC_TYPE_HISTOGRAM);
+    hdr_add(dest->data->hdr, src);
+
+    dest->operated_after_reset = true;
+}
+
 long long metric_histogram_value_at_percentile(const struct metric *pthis, double percentile) {
     return hdr_value_at_percentile(pthis->data->hdr, percentile);
 }
diff --git a/src/metrics/metric.h b/src/metrics/metric.h
index 0dee126..9a077a3 100644
--- a/src/metrics/metric.h
+++ b/src/metrics/metric.h
@@ -4,6 +4,7 @@
 #include <stdbool.h>
 #include "fieldstat.h" // for enum metric_type
 #include "metric_manifest.h"
+#include "hdr/hdr_histogram.h"
 
 struct metric;
 
@@ -27,4 +28,4 @@ int metric_histogram_record(struct metric *pthis, long long value);
 // for the meaning of these two query method, refer to https://prometheus.io/docs/concepts/metric_types/, and search for "histogram" and "summary"
 long long metric_histogram_value_at_percentile(const struct metric *pthis, double percentile);
 long long metric_histogram_count_le_value(const struct metric *pthis, long long value);
-
+void metric_histogram_merge(struct metric *dest, const struct hdr_histogram *src);
diff --git a/src/version.map b/src/version.map
index 5e22600..48a2365 100644
--- a/src/version.map
+++ b/src/version.map
@@ -1,5 +1,8 @@
 {
-  global: 	*fieldstat_*, *histogram*;
-	GIT_VERSION*;
-  local: *;
+    global:
+        *fieldstat_*;
+        *histogram*;
+        GIT_VERSION*;
+    local:
+        *;
 };
\ No newline at end of file
diff --git a/test/test_metric_histogram.cpp b/test/test_metric_histogram.cpp
index 39132d3..e82000e 100644
--- a/test/test_metric_histogram.cpp
+++ b/test/test_metric_histogram.cpp
@@ -191,6 +191,46 @@ TEST(metric_test_histogram, can_add_0value) // histogram only allow min_val > 0,
     fieldstat_free(instance);
 }
 
+TEST(metric_test_histogram, histogram_batch_transaction) {
+    struct fieldstat *instance = fieldstat_new();
+    int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+    int metric_id = fieldstat_register_histogram(instance, cube_id, "test_histogram_metric", 1, 10000, 5);
+
+    struct histogram *histogram = fieldstat_histogram_fork(instance, cube_id, metric_id);
+    ASSERT_NE(histogram, nullptr);
+
+    long long packet_lengths[] = {1234, 1234, 123, 567, 890, 1234, 1500};
+    size_t packet_count = sizeof(packet_lengths) / sizeof(packet_lengths[0]);
+
+    for (size_t i = 0; i < packet_count; ++i) {
+        int tmp_ret = histogram_record(histogram, packet_lengths[i]);
+        EXPECT_EQ(tmp_ret, 0);
+    }
+
+    int merge_result = fieldstat_histogram_merge(instance, cube_id, metric_id, &TEST_FIELD_STRING, 1, histogram);
+    EXPECT_EQ(merge_result, 0);
+
+    double percentile_value = fieldstat_histogram_value_at_percentile(instance, cube_id, &TEST_FIELD_LIST_STRING, metric_id, 50.0);
+    EXPECT_EQ(percentile_value, 1234);
+    int count_le = fieldstat_histogram_count_le_value(instance, cube_id, &TEST_FIELD_LIST_STRING, metric_id, 1000);
+    EXPECT_EQ(count_le, 3);
+
+    // test reset
+    histogram_reset(histogram);
+    histogram_record(histogram, 2000);
+    histogram_record(histogram, 2500);
+    merge_result = fieldstat_histogram_merge(instance, cube_id, metric_id, &TEST_FIELD_STRING, 1, histogram);
+    EXPECT_EQ(merge_result, 0);
+
+    percentile_value = fieldstat_histogram_value_at_percentile(instance, cube_id, &TEST_FIELD_LIST_STRING, metric_id, 95.0);
+    EXPECT_EQ(percentile_value, 2500);
+    count_le = fieldstat_histogram_count_le_value(instance, cube_id, &TEST_FIELD_LIST_STRING, metric_id, 1000);
+    EXPECT_EQ(count_le, 3);
+
+    histogram_free(histogram);
+    fieldstat_free(instance);
+}
+
 
 int main(int argc, char *argv[]) 
 {