diff options
| author | chenzizhan <[email protected]> | 2024-07-17 15:46:47 +0800 |
|---|---|---|
| committer | chenzizhan <[email protected]> | 2024-07-17 15:46:47 +0800 |
| commit | 8ce45ff4f96186c5d9e0f3e82addd2085d7c8788 (patch) | |
| tree | ccd28d85c7024d2dd4474ae8e6b873adca74b5eb | |
| parent | dccb4ce1fd92b1f142383e585487af08831264d3 (diff) | |
cube set sampling
| -rw-r--r-- | include/fieldstat/fieldstat.h | 13 | ||||
| -rw-r--r-- | readme_fieldstat.md | 2 | ||||
| -rw-r--r-- | src/cells/hash_table.c | 3 | ||||
| -rw-r--r-- | src/cells/heavy_keeper.c | 3 | ||||
| -rw-r--r-- | src/cells/spread_sketch.c | 3 | ||||
| -rw-r--r-- | src/cube.c | 86 | ||||
| -rw-r--r-- | src/cube.h | 3 | ||||
| -rw-r--r-- | src/fieldstat.c | 46 | ||||
| -rw-r--r-- | src/fieldstat_easy.c | 3 | ||||
| -rw-r--r-- | test/profiling/main.c | 2 | ||||
| -rw-r--r-- | test/test_empty_tags.cpp | 13 | ||||
| -rw-r--r-- | test/test_exporter_json.cpp | 57 | ||||
| -rw-r--r-- | test/test_fuzz_test.cpp | 37 | ||||
| -rw-r--r-- | test/test_merge.cpp | 70 | ||||
| -rw-r--r-- | test/test_metric_counter.cpp | 28 | ||||
| -rw-r--r-- | test/test_metric_histogram.cpp | 16 | ||||
| -rw-r--r-- | test/test_metric_hll.cpp | 17 | ||||
| -rw-r--r-- | test/test_performance.cpp | 39 | ||||
| -rw-r--r-- | test/test_register_and_reset.cpp | 138 | ||||
| -rw-r--r-- | test/test_write_json_file.cpp | 14 |
20 files changed, 383 insertions, 210 deletions
diff --git a/include/fieldstat/fieldstat.h b/include/fieldstat/fieldstat.h index b5fef15..0f77ec4 100644 --- a/include/fieldstat/fieldstat.h +++ b/include/fieldstat/fieldstat.h @@ -67,11 +67,8 @@ int fieldstat_calibrate(const struct fieldstat *master, struct fieldstat *replic * @param max_n_cell: max number of samplings(cells) in each cube. When mode is TOPK, max_n_cell > 0, while in COMPREHENSIVE mode, max_n_cell can be 0, meaning that there is no limit. * @return cube id, if success; otherwise, return FS_ERR_NULL_HANDLER, or FS_ERR_INVALID_PARAM when (max_n_cell == 0 && mode == TOPK). return FS_ERR_INVALID_KEY when the cube_dimensions is not unique. */ -int fieldstat_create_cube(struct fieldstat *instance, const struct field *cube_dimensions, size_t n_dimension, enum sampling_mode mode, size_t max_n_cell); -// todo: 重命名为fieldstat_cube_create - -//todo: create cube 接口变化 -// int fieldstat_cube_set_sampling(struct fieldstat *instance, int cube_id, enum sampling_mode mode, int max_n_cell, int primary_metric_id); +int fieldstat_cube_create(struct fieldstat *instance, const struct field *cube_dimensions, size_t n_dimension); +int fieldstat_cube_set_sampling(struct fieldstat *instance, int cube_id, enum sampling_mode mode, int max_n_cell, int primary_metric_id); /* @brief Change the topk cube primary metric id. When fieldstat_counter_add or fieldstat_counter_set are called on the primary metric, the topk record of such cell will be updated. @@ -87,7 +84,7 @@ int fieldstat_cube_set_primary_metric(struct fieldstat *instance, int cube_id, i * @brief Delete the cube of cube_id. All the cells and metrics are deleted. The cube_id may be reused by other new cubes. Increase the corresponding cube_version by 1. * @return FS_OK, FS_ERR_NULL_HANDLER or FS_ERR_INVALID_CUBE_ID */ -int fieldstat_destroy_cube(struct fieldstat *instance, int cube_id); +int fieldstat_cube_destroy(struct fieldstat *instance, int cube_id); /* * @brief add a metric to the cube of cube_id. One metric may be associated with different cells. @@ -114,7 +111,7 @@ int fieldstat_register_histogram(struct fieldstat *instance, int cube_id, const /* * @brief let the value of counter metric of cell_id increase by `increment`. - * @param cube_id: cube id, previously returned by fieldstat_create_cube. + * @param cube_id: cube id, previously returned by fieldstat_cube_create. * @param metric_id: metric id, previously returned by fieldstat_register_counter. * @param increment: increment of the counter metric. Can be negative. * @return FS_OK if success. FS_ERR_NULL_HANDLER, FS_ERR_INVALID_CUBE_ID, FS_ERR_INVALID_METRIC_ID if fail. @@ -200,7 +197,7 @@ enum metric_type fieldstat_get_metric_type(const struct fieldstat *instance, int void fieldstat_cube_get_cells(const struct fieldstat *instance, int cube_id, struct field_list **cell_dimensions, size_t *n_cell); /* - get the field of fieldstat_create_cube. User free them by calling fieldstat_tag_list_arr_free(struct field_list *, 1) + get the field of fieldstat_cube_create. User free them by calling fieldstat_tag_list_arr_free(struct field_list *, 1) return NULL when ID is invalid. */ struct field_list *fieldstat_cube_get_tags(const struct fieldstat *instance, int cube_id); diff --git a/readme_fieldstat.md b/readme_fieldstat.md index 42c6680..4dd7a99 100644 --- a/readme_fieldstat.md +++ b/readme_fieldstat.md @@ -40,7 +40,7 @@ Download fieldstat4 rpm from https://repo.geedge.net/pulp/content/ and install r #include "fieldstat.h" struct fieldstat *instance = fieldstat_new(); -int cube_id = fieldstat_create_cube(instance, YOUR_SHARED_TAG, YOUR_SHARED_TAG_LENGTH, SAMPLING_MODE_TOPK, MAX_CELL_NUMBER); +int cube_id = fieldstat_cube_create(instance, YOUR_SHARED_TAG, YOUR_SHARED_TAG_LENGTH, SAMPLING_MODE_TOPK, MAX_CELL_NUMBER); int metric_counter_id = fieldstat_register_counter(instance, cube_id, "any metric name", 0/1); int metric_histogram_id = fieldstat_register_histogramogram(instance, cube_id, "any metric name", THE_MINIMUM_NUMBER_TO_RECORD, THE_MAXIMUM_NUMBER_TO_RECORD, PRECISION); int metric_hll_id = fieldstat_register_hll(instance, cube_id, "any metric name", PRECISION); diff --git a/src/cells/hash_table.c b/src/cells/hash_table.c index d6dbfb5..60d2465 100644 --- a/src/cells/hash_table.c +++ b/src/cells/hash_table.c @@ -64,6 +64,9 @@ struct hash_table *hash_table_new(int max_query_num) { } void hash_table_free(struct hash_table *pthis) { + if (pthis == NULL) { + return; + } struct tag_exdata_item *item, *tmp; HASH_ITER(hh, pthis->tag_id_map, item, tmp) { HASH_DEL(pthis->tag_id_map, item); diff --git a/src/cells/heavy_keeper.c b/src/cells/heavy_keeper.c index b606a67..c62e550 100644 --- a/src/cells/heavy_keeper.c +++ b/src/cells/heavy_keeper.c @@ -518,6 +518,9 @@ struct heavy_keeper *heavy_keeper_new(int max_query_num) { } void heavy_keeper_free(struct heavy_keeper *hk) { + if (hk == NULL) { + return; + } sorted_set_free(hk->top_K_heap); free(hk->sketch); diff --git a/src/cells/spread_sketch.c b/src/cells/spread_sketch.c index 5d5dd72..3e9b11e 100644 --- a/src/cells/spread_sketch.c +++ b/src/cells/spread_sketch.c @@ -382,6 +382,9 @@ struct spread_sketch *duplicate_and_step(const struct spread_sketch *ss, const s } void spread_sketch_free(struct spread_sketch *ss) { + if (ss == NULL) { + return; + } smart_ptr_table_free(ss->table); for (int i = 0; i < ss->depth * ss->width; i++) { hll_free_register(ss->buckets[i].sthll_register); @@ -497,10 +497,9 @@ void *exdata_copy_i(void *exdata) { return cell_copy((struct cell *)exdata); } -struct cube *cube_info_new(const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, size_t max_n_cell) +struct cube *cube_info_new(const struct field *dimensions, size_t n_dimensions) { struct cube *cube = calloc(1, sizeof(struct cube)); - cube->sampling_mode = mode; if (n_dimensions == 0) { cube->cube_dimensions = NULL; @@ -509,42 +508,81 @@ struct cube *cube_info_new(const struct field *dimensions, size_t n_dimensions, } cube->n_dimensions = n_dimensions; - cube->max_n_cell = max_n_cell; field_array_to_key(dimensions, n_dimensions, &cube->serialized_dimensions, &cube->serialized_dimensions_len); cube->id = -1; + cube->primary_metric_id = -1; return cube; } -struct cube *cube_new(const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, size_t max_n_cell) +struct cube *cube_new(const struct field *dimensions, size_t n_dimensions) { - struct cube *cube = cube_info_new(dimensions, n_dimensions, mode, max_n_cell); + struct cube *cube = cube_info_new(dimensions, n_dimensions); cube->manifest_manager = metric_manifest_manager_new(); - switch (mode) - { + return cube; +} + +int cube_set_sampling_mode(struct cube *cube, enum sampling_mode mode, int max_n_cell, int primary_metric_id) { + if (cube->sampling_mode == mode && cube->max_n_cell == max_n_cell && cube->primary_metric_id == primary_metric_id) { + return FS_OK; + } + const struct metric_manifest *manifest = metric_manifest_manager_get_by_id(cube->manifest_manager, primary_metric_id); + if (manifest == NULL && mode != SAMPLING_MODE_COMPREHENSIVE) { + return FS_ERR_INVALID_METRIC_ID; + } + if ((mode == SAMPLING_MODE_TOPK && manifest->type != METRIC_TYPE_COUNTER) || + (mode == SAMPLING_MODE_TOP_CARDINALITY && manifest->type != METRIC_TYPE_HLL)) { + return FS_ERR_INVALID_PARAM; + } + + if (cube->primary_metric_id != -1) { + // delete previous settings + switch (cube->sampling_mode) + { case SAMPLING_MODE_TOPK: - cube->heavykeeper = heavy_keeper_new(max_n_cell); - heavy_keeper_set_exdata_schema(cube->heavykeeper, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i); + heavy_keeper_free(cube->heavykeeper); break; case SAMPLING_MODE_COMPREHENSIVE: - cube->table = hash_table_new(max_n_cell); - hash_table_set_exdata_schema(cube->table, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i); + hash_table_free(cube->table); + break; + case SAMPLING_MODE_TOP_CARDINALITY: + spread_sketch_free(cube->spread_sketch); break; - case SAMPLING_MODE_TOP_CARDINALITY: { - int width, depth; - unsigned char precision; - spread_sketch_get_parameter_recommendation(max_n_cell, &depth, &width, &precision); - cube->spread_sketch = spread_sketch_new(depth, width, precision, 0, DUMMY_TIME_VAL); - spread_sketch_set_exdata_schema(cube->spread_sketch, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i); - break; } default: assert(0); break; + } } - return cube; + switch (mode) + { + case SAMPLING_MODE_TOPK: + cube->heavykeeper = heavy_keeper_new(max_n_cell); + heavy_keeper_set_exdata_schema(cube->heavykeeper, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i); + break; + case SAMPLING_MODE_COMPREHENSIVE: + cube->table = hash_table_new(max_n_cell); + hash_table_set_exdata_schema(cube->table, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i); + break; + case SAMPLING_MODE_TOP_CARDINALITY: { + int width, depth; + unsigned char precision; + spread_sketch_get_parameter_recommendation(max_n_cell, &depth, &width, &precision); + cube->spread_sketch = spread_sketch_new(depth, width, precision, 0, DUMMY_TIME_VAL); + spread_sketch_set_exdata_schema(cube->spread_sketch, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i); + break; } + default: + assert(0); + break; + } + + cube->sampling_mode = mode; + cube->max_n_cell = max_n_cell; + cube->primary_metric_id = primary_metric_id; + + return FS_OK; } void cube_free(struct cube *cube) { @@ -932,6 +970,7 @@ int cube_hll_add_field(struct cube *cube, int metric_id, const struct field *dim } int cube_counter_incrby(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, long long increment) { + assert(cube->primary_metric_id != -1); assert(cube->sampling_mode == SAMPLING_MODE_COMPREHENSIVE || (cube->sampling_mode == SAMPLING_MODE_TOPK && (cube->primary_metric_id != metric_id || increment >= 0)) || (cube->sampling_mode == SAMPLING_MODE_TOP_CARDINALITY && cube->primary_metric_id != metric_id) @@ -1020,8 +1059,10 @@ int cube_counter_set(struct cube *cube, int metric_id, const struct field *dimen struct cube *cube_copy(const struct cube *cube) { - struct cube *cube_dup = cube_info_new(cube->cube_dimensions, cube->n_dimensions, cube->sampling_mode, cube->max_n_cell); + struct cube *cube_dup = cube_info_new(cube->cube_dimensions, cube->n_dimensions); cube_dup->primary_metric_id = cube->primary_metric_id; + cube_dup->sampling_mode = cube->sampling_mode; + cube_dup->max_n_cell = cube->max_n_cell; switch (cube->sampling_mode) { @@ -1087,8 +1128,10 @@ int cube_merge(struct cube *dest, const struct cube *src) } struct cube *cube_fork(const struct cube *cube) { - struct cube *ret = cube_info_new(cube->cube_dimensions, cube->n_dimensions, cube->sampling_mode, cube->max_n_cell); + struct cube *ret = cube_info_new(cube->cube_dimensions, cube->n_dimensions); ret->primary_metric_id = cube->primary_metric_id; + ret->sampling_mode = cube->sampling_mode; + ret->max_n_cell = cube->max_n_cell; ret->manifest_manager = metric_manifest_manager_copy(cube->manifest_manager); switch (cube->sampling_mode) { @@ -1431,7 +1474,6 @@ struct field_list *cube_get_identifier(const struct cube *cube) { const char *cube_get_metric_name(const struct cube *cube, int metric_id) { const struct metric_manifest *metric = metric_manifest_manager_get_by_id(cube->manifest_manager, metric_id); if (metric == NULL) { - printf("metric is null\n"); return NULL; } @@ -13,7 +13,8 @@ extern "C" struct cube; struct cube_manager; -struct cube *cube_new(const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, size_t max_n_cell); +struct cube *cube_new(const struct field *dimensions, size_t n_dimensions); +int cube_set_sampling_mode(struct cube *cube, enum sampling_mode mode, int max_n_cell, int primary_metric_id); void cube_free(struct cube *cube); void cube_reset(struct cube *cube); struct cube *cube_copy(const struct cube *cube); diff --git a/src/fieldstat.c b/src/fieldstat.c index 4f4582b..002d754 100644 --- a/src/fieldstat.c +++ b/src/fieldstat.c @@ -48,7 +48,7 @@ void fieldstat_reset(struct fieldstat *instance) cube_manager_reset(instance->cube_manager); } -int fieldstat_destroy_cube(struct fieldstat *instance, int cube_id) +int fieldstat_cube_destroy(struct fieldstat *instance, int cube_id) { struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); if (cube == NULL) { @@ -76,7 +76,24 @@ void fieldstat_free_tag_array(struct field *fields, size_t n_tags) free(fields); } -int fieldstat_create_cube(struct fieldstat *instance, const struct field *cube_dimensions, size_t n_dimension, enum sampling_mode mode, size_t max_n_cell) +// cppcheck-suppress [constParameterPointer, unmatchedSuppression] +int fieldstat_cube_set_sampling(struct fieldstat *instance, int cube_id, enum sampling_mode mode, int max_n_cell, int primary_metric_id) { + if (max_n_cell <= 0) { + if (mode != SAMPLING_MODE_COMPREHENSIVE) { + return FS_ERR_INVALID_PARAM; + } + max_n_cell = INT32_MAX; + } + + struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + if (cube == NULL) { + return FS_ERR_INVALID_CUBE_ID; + } + + return cube_set_sampling_mode(cube, mode, max_n_cell, primary_metric_id); +} + +int fieldstat_cube_create(struct fieldstat *instance, const struct field *cube_dimensions, size_t n_dimension) { if (instance == NULL) { return FS_ERR_NULL_HANDLER; @@ -86,14 +103,8 @@ int fieldstat_create_cube(struct fieldstat *instance, const struct field *cube_d cube_dimensions = NULL; n_dimension = 0; } - if (mode == SAMPLING_MODE_TOPK && max_n_cell == 0) { - return FS_ERR_INVALID_PARAM; - } - if (max_n_cell == 0) { - max_n_cell = INT32_MAX; - } - - struct cube *cube = cube_new(cube_dimensions, n_dimension, mode, max_n_cell); + + struct cube *cube = cube_new(cube_dimensions, n_dimension); int ret = cube_manager_add(instance->cube_manager, cube); if (ret < 0) { cube_free(cube); @@ -103,21 +114,6 @@ int fieldstat_create_cube(struct fieldstat *instance, const struct field *cube_d return ret; //ret is the cube_id } -// cppcheck-suppress [constParameterPointer, unmatchedSuppression] -int fieldstat_cube_set_primary_metric(struct fieldstat *instance, int cube_id, int metric_id) -{ - if (instance == NULL) { - return FS_ERR_NULL_HANDLER; - } - - struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); - if (cube == NULL) { - return FS_ERR_INVALID_CUBE_ID; - } - - return cube_set_primary_metric(cube, metric_id); -} - /* -------------------------------------------------------------------------- */ /* metric register */ /* -------------------------------------------------------------------------- */ diff --git a/src/fieldstat_easy.c b/src/fieldstat_easy.c index 00a569a..46133f4 100644 --- a/src/fieldstat_easy.c +++ b/src/fieldstat_easy.c @@ -122,7 +122,8 @@ struct fieldstat_easy *fieldstat_easy_new(int max_thread_num, const char *name, fse->fsu = malloc(sizeof(struct fs_easy_thread) * max_thread_num); fse->max_thread_num = max_thread_num; fse->delta = fieldstat_new(); - fieldstat_create_cube(fse->delta, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); + fieldstat_cube_create(fse->delta, NULL, 0); + fieldstat_cube_set_sampling(fse->delta, 0, SAMPLING_MODE_COMPREHENSIVE, 0, 0); fse->accumulate = fieldstat_fork(fse->delta); fse->exporter = fieldstat_json_exporter_new(); diff --git a/test/profiling/main.c b/test/profiling/main.c index 3af2b1a..8be10d3 100644 --- a/test/profiling/main.c +++ b/test/profiling/main.c @@ -31,7 +31,7 @@ int main () { }; struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); + fieldstat_cube_create(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); fieldstat_register_counter(instance, "counter"); start = clock(); diff --git a/test/test_empty_tags.cpp b/test/test_empty_tags.cpp index 967f8c0..8360eab 100644 --- a/test/test_empty_tags.cpp +++ b/test/test_empty_tags.cpp @@ -20,13 +20,13 @@ void assert_cell_null(const struct fieldstat *instance, int cube_id, long long e fieldstat_tag_list_arr_free(tag_list, n_cell); } - TEST(test_empty_tag, add_many_times) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 1); - + int cube_id = fieldstat_cube_create(instance, NULL, 0); int metric_id = fieldstat_register_counter(instance, cube_id, "metric"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_COMPREHENSIVE, 1, 0); + fieldstat_counter_incrby(instance, cube_id, metric_id, NULL, 0, 1); fieldstat_counter_incrby(instance, cube_id, metric_id, NULL, 0, 1); @@ -38,8 +38,10 @@ TEST(test_empty_tag, add_many_times) struct fieldstat *test_empty_my_init(enum sampling_mode mode = SAMPLING_MODE_COMPREHENSIVE) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, NULL, 0, mode, 1); + int cube_id = fieldstat_cube_create(instance, NULL, 0); int metric_id = fieldstat_register_counter(instance, cube_id, "metric"); + fieldstat_cube_set_sampling(instance, cube_id, mode, 1, 0); + fieldstat_counter_incrby(instance, cube_id, metric_id, NULL, 0, 1); return instance; @@ -97,8 +99,9 @@ TEST(test_empty_tag, merge_topk) TEST(test_empty_tag, merge_spreadsketch) { struct fieldstat *instance_src = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance_src, NULL, 0, SAMPLING_MODE_TOP_CARDINALITY, 1); + int cube_id = fieldstat_cube_create(instance_src, NULL, 0); int metric_id = fieldstat_register_hll(instance_src, cube_id, "metric", 4); + fieldstat_cube_set_sampling(instance_src, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 1, 0); fieldstat_hll_add(instance_src, cube_id, metric_id, NULL, 0, "1", 1); struct fieldstat *instance_dst = fieldstat_new(); diff --git a/test/test_exporter_json.cpp b/test/test_exporter_json.cpp index 45a05cf..ebf6750 100644 --- a/test/test_exporter_json.cpp +++ b/test/test_exporter_json.cpp @@ -167,9 +167,10 @@ void topk_standard_oper(const std::function<void(Fieldstat_tag_list_wrapper *, u void topk_init(struct fieldstat *instance, unsigned int test_expected_big_count) { const char *metric_name[TEST_METRIC_NUM] = {"topk1", "topk2"}; - int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_TOPK, TEST_TOPK_STANDARD_K); + int cube_id = fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3); int m1 = fieldstat_register_counter(instance, cube_id, metric_name[0]); int m2 = fieldstat_register_counter(instance, cube_id, metric_name[1]); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, TEST_TOPK_STANDARD_K, 0); std::function<void(Fieldstat_tag_list_wrapper *, unsigned int *)> topk_add = [instance, cube_id, m1, m2](const Fieldstat_tag_list_wrapper *my_tags, unsigned int counts[TEST_METRIC_NUM]) { @@ -186,12 +187,14 @@ TEST(export_test, cjson_export_with_fixed_tag_and_many_metrics_on_one_cube_of_co // new instance struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, tag_list_num); + int cube_id = fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3); int id_counter = fieldstat_register_counter(instance, cube_id, "counter"); int id_gauge = fieldstat_register_hll(instance, cube_id, "gauge", g_hll_standard->cfg.precision); int id_histogram = fieldstat_register_histogram(instance, cube_id, "histogram", g_histogram_standard->lowest_discernible_value, g_histogram_standard->highest_trackable_value, g_histogram_standard->significant_figures); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_COMPREHENSIVE, tag_list_num, 0); + Fieldstat_tag_list_wrapper *fields[tag_list_num]; fill_random_tag(fields, tag_list_num); @@ -289,9 +292,11 @@ extern "C" { TEST(export_test, cjson_export_on_one_cube_of_spreadsketch_sampling) { int K = 10; struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, K); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id = fieldstat_register_hll(instance, cube_id, "metric", 7); int metric_count = fieldstat_register_counter(instance, cube_id, "oper cnt"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, K, 0); + SpreadSketchZipfGenerator flow_generator(1.0, K * 10); for (int i = 0; i < 100000; i++) { @@ -342,11 +347,19 @@ TEST(export_test, empty_fieldstat_export_null) { fieldstat_free(instance); } +int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k, int primary_metric_id=0) +{ + assert(mode == SAMPLING_MODE_COMPREHENSIVE); + int ret = fieldstat_cube_create(instance, dimensions, n_dimensions); + fieldstat_cube_set_sampling(instance, ret, mode, k, primary_metric_id); + return ret; +} + TEST(export_test, only_registered_but_not_added_export_null_with_global_tag) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3); + int cube_id = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3); fieldstat_register_counter(instance, cube_id, "counter"); fieldstat_register_hll(instance, cube_id, "gauge", g_hll_standard->cfg.precision); fieldstat_register_histogram(instance, cube_id, "histogram", @@ -362,9 +375,9 @@ TEST(export_test, only_registered_but_not_added_export_null_with_global_tag) TEST(export_test, skip_two_empty_cube_and_export_last_one_with_global_tag) { struct fieldstat *instance = fieldstat_new(); - (void)fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3); - (void)fieldstat_create_cube(instance, TEST_TAG_SHARED2, 3, SAMPLING_MODE_COMPREHENSIVE, 3); - int cube_id_3 = fieldstat_create_cube(instance, TEST_TAG_SHARED3, 1, SAMPLING_MODE_COMPREHENSIVE, 3); + (void)test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3); + (void)test_fieldstat_cube_create(instance, TEST_TAG_SHARED2, 3, SAMPLING_MODE_COMPREHENSIVE, 3); + int cube_id_3 = test_fieldstat_cube_create(instance, TEST_TAG_SHARED3, 1, SAMPLING_MODE_COMPREHENSIVE, 3); (void)fieldstat_register_counter(instance, cube_id_3, "counter"); (void)fieldstat_register_hll(instance, cube_id_3, "gauge", g_hll_standard->cfg.precision); int id_histogram = fieldstat_register_histogram(instance, cube_id_3, "histogram", @@ -410,9 +423,9 @@ TEST(export_test, skip_two_empty_cube_and_export_last_one_with_global_tag) TEST(export_test, skip_empty_metrics_given_cube_deleted) { struct fieldstat *instance = fieldstat_new(); - int cube_id_del = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3); - int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED2, 3, SAMPLING_MODE_COMPREHENSIVE, 3); - fieldstat_destroy_cube(instance, cube_id_del); + int cube_id_del = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3); + int cube_id = test_fieldstat_cube_create(instance, TEST_TAG_SHARED2, 3, SAMPLING_MODE_COMPREHENSIVE, 3); + fieldstat_cube_destroy(instance, cube_id_del); (void)fieldstat_register_counter(instance, cube_id, "counter"); (void)fieldstat_register_counter(instance, cube_id, "counter2"); int metric_id = fieldstat_register_counter(instance, cube_id, "counter3"); @@ -459,7 +472,7 @@ TEST(export_test, skip_empty_metrics_given_cube_deleted) { TEST(export_test, enable_delta_and_export_twice_without_new_metric) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10); int id_counter = fieldstat_register_counter(instance, cube_id, "counter"); fieldstat_counter_incrby(instance, cube_id, id_counter, &TEST_TAG_INT, 1, 1); @@ -528,7 +541,7 @@ TEST(export_test, enable_delta_and_export_twice_without_new_metric) TEST(export_test, enable_delta_and_export_twice_with_new_metric_and_omit_histogram) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10); int id_counter = fieldstat_register_counter(instance, cube_id, "counter"); int id_histogram = fieldstat_register_histogram(instance, cube_id, "histogram", g_histogram_standard->lowest_discernible_value, g_histogram_standard->highest_trackable_value, g_histogram_standard->significant_figures); @@ -610,7 +623,7 @@ TEST(export_test, enable_delta_and_export_twice_with_new_metric_and_omit_histogr TEST(export_test, enable_delta_and_export_three_times_skipping_cube_with_no_counter) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10); int id_counter = fieldstat_register_counter(instance, cube_id, "counter"); int id_histogram = fieldstat_register_histogram(instance, cube_id, "histogram", g_histogram_standard->lowest_discernible_value, g_histogram_standard->highest_trackable_value, g_histogram_standard->significant_figures); @@ -719,12 +732,12 @@ void test_check_delta_for_one_json(const struct field_list *expect_cell_tag, con TEST(export_test, enable_delta_and_export_instance_with_many_cells_with_global_tags) { struct fieldstat *instance = fieldstat_new(); - int cube_id1 = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id1 = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10); int id_counter1 = fieldstat_register_counter(instance, cube_id1, "counter"); fieldstat_counter_incrby(instance, cube_id1, id_counter1, &TEST_TAG_INT, 1, 11); fieldstat_counter_incrby(instance, cube_id1, id_counter1, &TEST_TAG_STRING, 1, 12); - int cube_id2 = fieldstat_create_cube(instance, TEST_TAG_SHARED3, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id2 = test_fieldstat_cube_create(instance, TEST_TAG_SHARED3, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int id_counter2 = fieldstat_register_counter(instance, cube_id2, "counter"); fieldstat_counter_incrby(instance, cube_id2, id_counter2, &TEST_TAG_INT, 1, 21); fieldstat_counter_incrby(instance, cube_id2, id_counter2, &TEST_TAG_DOUBLE, 1, 22); @@ -805,7 +818,7 @@ TEST(export_test, enable_delta_and_export_instance_with_many_cells_with_global_t void test_reset_one_round(std::function<void(struct fieldstat *, struct fieldstat_json_exporter *)> trigger_reset) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 1); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 1); int id_counter = fieldstat_register_counter(instance, cube_id, "counter"); fieldstat_counter_incrby(instance, cube_id, id_counter, &TEST_TAG_INT, 1, 11); @@ -869,8 +882,8 @@ TEST(export_test, enable_delta_and_reset_on_change_exporter_tag) { TEST(export_test, enable_delta_and_reset_on_delete_cube) { auto trigger = [](struct fieldstat *instance, struct fieldstat_json_exporter *fieldstat_json_exporter) { - fieldstat_destroy_cube(instance, 0); - fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3); + fieldstat_cube_destroy(instance, 0); + test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3); fieldstat_register_counter(instance, 0, "counter"); }; @@ -880,7 +893,7 @@ TEST(export_test, enable_delta_and_reset_on_delete_cube) { TEST(export_test, delta_with_two_instance_same_config) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); + int cube_id = test_fieldstat_cube_create(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); int id_counter = fieldstat_register_counter(instance, cube_id, "counter"); fieldstat_counter_incrby(instance, 0, id_counter, &TEST_TAG_INT, 1, 123); int id_hist = fieldstat_register_histogram(instance, cube_id, "histogram", 1, 1000, 3); @@ -920,7 +933,7 @@ TEST(export_test, delta_with_two_instance_same_config) TEST(export_test, delta_with_two_instance_one_empty) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); + int cube_id = test_fieldstat_cube_create(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); int id_counter = fieldstat_register_counter(instance, cube_id, "counter"); fieldstat_counter_incrby(instance, 0, id_counter, &TEST_TAG_INT, 1, 123); int id_hist = fieldstat_register_histogram(instance, cube_id, "histogram", 1, 1000, 3); @@ -959,7 +972,7 @@ TEST(export_test, delta_with_two_instance_one_empty) TEST(export_test, delta_with_two_instance_different_cell) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); + test_fieldstat_cube_create(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); int id_counter = fieldstat_register_counter(instance, 0, "counter"); fieldstat_counter_incrby(instance, 0, id_counter, &TEST_TAG_INT, 1, 123); @@ -1022,7 +1035,7 @@ TEST(export_test, export_flat_null_with_only_global_tag) { TEST(export_test, export_flat_many_tags_many_cell) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 3); + int cube_id = test_fieldstat_cube_create(instance, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 3); int metric_id = fieldstat_register_counter(instance, cube_id, "counter metric"); diff --git a/test/test_fuzz_test.cpp b/test/test_fuzz_test.cpp index 9eed5ed..aa3487a 100644 --- a/test/test_fuzz_test.cpp +++ b/test/test_fuzz_test.cpp @@ -77,11 +77,12 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_ // init cube for (int i = 0; i < CUBE_NUM; i++) { shared_tags[i] = new Fieldstat_tag_list_wrapper("shared_tag", i); - int cube_id = fieldstat_create_cube(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count(), SAMPLING_MODE_COMPREHENSIVE, CELL_MAX); + int cube_id = fieldstat_cube_create(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count()); EXPECT_EQ(cube_id, i); fieldstat_register_counter(master, cube_id, metric_name[METRIC_ID_COUNTER]); fieldstat_register_hll(master, cube_id, metric_name[METRIC_ID_HLL], 6); + fieldstat_cube_set_sampling(master, cube_id, SAMPLING_MODE_COMPREHENSIVE, CELL_MAX, 0); } // all the possible fields @@ -120,10 +121,12 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_ Fieldstat_tag_list_wrapper *new_tag = new Fieldstat_tag_list_wrapper("shared_tag", next_shared_tag_value++); delete shared_tags[cube_id_to_change]; shared_tags[cube_id_to_change] = new_tag; - fieldstat_destroy_cube(master, cube_id_to_change); - int cube_id_new = fieldstat_create_cube(master, new_tag->get_tag(), new_tag->get_tag_count(), SAMPLING_MODE_COMPREHENSIVE, CELL_MAX); + fieldstat_cube_destroy(master, cube_id_to_change); + int cube_id_new = fieldstat_cube_create(master, new_tag->get_tag(), new_tag->get_tag_count()); fieldstat_register_counter(master, cube_id_new, metric_name[METRIC_ID_COUNTER]); fieldstat_register_hll(master, cube_id_new, metric_name[METRIC_ID_HLL], 6); + fieldstat_cube_set_sampling(master, cube_id_new, SAMPLING_MODE_COMPREHENSIVE, CELL_MAX, 0); + EXPECT_EQ(cube_id_new, cube_id_to_change); // should new the cube in the hole leaved by the destroyed cube // calibrate for (int j = 0; j < INSTANCE_NUM; j++) { @@ -220,9 +223,10 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_ // init cube for (int i = 0; i < CUBE_NUM; i++) { shared_tags[i] = new Fieldstat_tag_list_wrapper("shared_tag", i); - int cube_id = fieldstat_create_cube(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count(), SAMPLING_MODE_TOPK, CELL_MAX); + int cube_id = fieldstat_cube_create(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count()); EXPECT_EQ(cube_id, i); fieldstat_register_counter(master, cube_id, "topk"); + fieldstat_cube_set_sampling(master, cube_id, SAMPLING_MODE_TOPK, CELL_MAX, 0); } // all the possible fields @@ -258,9 +262,11 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_ Fieldstat_tag_list_wrapper *new_tag = new Fieldstat_tag_list_wrapper("shared_tag", next_shared_tag_value++); delete shared_tags[cube_id_to_change]; shared_tags[cube_id_to_change] = new_tag; - fieldstat_destroy_cube(master, cube_id_to_change); - int cube_id_new = fieldstat_create_cube(master, new_tag->get_tag(), new_tag->get_tag_count(), SAMPLING_MODE_TOPK, CELL_MAX); + fieldstat_cube_destroy(master, cube_id_to_change); + int cube_id_new = fieldstat_cube_create(master, new_tag->get_tag(), new_tag->get_tag_count()); fieldstat_register_counter(master, cube_id_new, "topk"); + fieldstat_cube_set_sampling(master, cube_id_new, SAMPLING_MODE_TOPK, CELL_MAX, 0); + EXPECT_EQ(cube_id_new, cube_id_to_change); // should new the cube in the hole leaved by the destroyed cube // calibrate for (int j = 0; j < INSTANCE_NUM; j++) { @@ -361,9 +367,10 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_ // init cube for (int i = 0; i < CUBE_NUM; i++) { shared_tags[i] = new Fieldstat_tag_list_wrapper("shared_tag", i); - int cube_id = fieldstat_create_cube(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count(), SAMPLING_MODE_TOP_CARDINALITY, CELL_MAX); + int cube_id = fieldstat_cube_create(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count()); EXPECT_EQ(cube_id, i); fieldstat_register_hll(master, cube_id, "hll", 6); + fieldstat_cube_set_sampling(master, cube_id, SAMPLING_MODE_TOP_CARDINALITY, CELL_MAX, 0); } //init instance @@ -392,10 +399,12 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_ Fieldstat_tag_list_wrapper *new_tag = new Fieldstat_tag_list_wrapper("shared_tag", next_shared_tag_value++); delete shared_tags[cube_id_to_change]; shared_tags[cube_id_to_change] = new_tag; - fieldstat_destroy_cube(master, cube_id_to_change); - int cube_id_new = fieldstat_create_cube(master, new_tag->get_tag(), new_tag->get_tag_count(), SAMPLING_MODE_TOP_CARDINALITY, CELL_MAX); + fieldstat_cube_destroy(master, cube_id_to_change); + int cube_id_new = fieldstat_cube_create(master, new_tag->get_tag(), new_tag->get_tag_count()); fieldstat_register_hll(master, cube_id_new, "hll", 6); EXPECT_EQ(cube_id_new, cube_id_to_change); // should new the cube in the hole leaved by the destroyed cube + fieldstat_cube_set_sampling(master, cube_id_new, SAMPLING_MODE_TOP_CARDINALITY, CELL_MAX, 0); + // calibrate for (int j = 0; j < INSTANCE_NUM; j++) { fieldstat_calibrate(master, replica[j]); @@ -499,9 +508,11 @@ TEST(Fuzz_test, add_and_reset_with_randomly_generated_flows_and_randomly_chosen_ struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance,NULL,0,SAMPLING_MODE_TOPK, 1); // K = 1, just to increase the possibility of FS_ERR_TOO_MANY_CELLS + int cube_id = fieldstat_cube_create(instance,NULL,0); int primary_metric_id = fieldstat_register_counter(instance, cube_id, "counter"); int counter2_id = fieldstat_register_counter(instance, cube_id, "counter2"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 1, 0); // K = 1, just to increase the possibility of FS_ERR_TOO_MANY_CELLS + fieldstat_counter_incrby(instance, cube_id, primary_metric_id, tag_list_wrapper[0]->get_tag(), tag_list_wrapper[0]->get_tag_count(), 1); fieldstat_counter_incrby(instance, cube_id, counter2_id, tag_list_wrapper[0]->get_tag(), tag_list_wrapper[0]->get_tag_count(), 1); @@ -550,9 +561,10 @@ TEST(perf, simple_one_for_perf_topk) // init cube for (int i = 0; i < CUBE_NUM; i++) { shared_tags[i] = new Fieldstat_tag_list_wrapper("shared_tag", i); - int cube_id = fieldstat_create_cube(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count(), SAMPLING_MODE_TOPK, CELL_MAX); + int cube_id = fieldstat_cube_create(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count()); EXPECT_EQ(cube_id, i); fieldstat_register_counter(master, cube_id, "topk"); + fieldstat_cube_set_sampling(master, cube_id, SAMPLING_MODE_TOPK, CELL_MAX, 0); } // init metric @@ -598,8 +610,9 @@ TEST(perf, simple_one_for_perf_spreadsketch) const int TEST_ROUND = 500000; struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_TAG_STRING, 1, SAMPLING_MODE_TOP_CARDINALITY, CELL_MAX); + int cube_id = fieldstat_cube_create(instance, &TEST_TAG_STRING, 1); fieldstat_register_hll(instance, cube_id, "hll", 6); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, CELL_MAX, 0); SpreadSketchZipfGenerator generator(1.0, CELL_MAX * 10); Fieldstat_tag_list_wrapper *cell_dimension[TEST_ROUND]; diff --git a/test/test_merge.cpp b/test/test_merge.cpp index 6e8f802..5024cff 100644 --- a/test/test_merge.cpp +++ b/test/test_merge.cpp @@ -46,15 +46,23 @@ double merge_test_fieldstat_hll_get(const struct fieldstat *instance, int cube_i return ret; } +int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k, int primary_metric_id=0) +{ + assert(mode == SAMPLING_MODE_COMPREHENSIVE); + int ret = fieldstat_cube_create(instance, dimensions, n_dimensions); + fieldstat_cube_set_sampling(instance, ret, mode, k, primary_metric_id); + return ret; +} + TEST(unit_test_merge, cube_shared_tag_mapping_with_new_cube) { struct fieldstat *instance = fieldstat_new(); - (void)fieldstat_create_cube(instance, &TEST_TAG_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int cube_id2 = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + (void)test_fieldstat_cube_create(instance, &TEST_TAG_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id2 = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int metric_id = fieldstat_register_counter(instance,cube_id2,"metric in cube 2"); fieldstat_counter_incrby(instance, cube_id2, metric_id, &TEST_TAG_STRING, 1, 1); struct fieldstat *instance_dest = fieldstat_new(); - int cube_id_dest = fieldstat_create_cube(instance_dest, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id_dest = test_fieldstat_cube_create(instance_dest, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); fieldstat_merge(instance_dest, instance); @@ -87,7 +95,7 @@ TEST(unit_test_merge, empty_instance) TEST(unit_test_merge, new_cube_and_metric_to_empty_comprehensive) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + test_fieldstat_cube_create(instance, &TEST_TAG_INT, 1, SAMPLING_MODE_COMPREHENSIVE, 10); fieldstat_register_counter(instance, 0, "metric_name"); struct fieldstat *instance_dest = fieldstat_new(); @@ -108,7 +116,7 @@ TEST(unit_test_merge, new_cube_and_metric_to_empty_comprehensive) TEST(unit_test_merge, new_cell_on_existing_cube_and_metric_comprehensive) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int metric_id = fieldstat_register_counter(instance, 0, "metric_name"); struct fieldstat *instance_dest = fieldstat_new(); fieldstat_merge(instance_dest, instance); @@ -140,7 +148,7 @@ TEST(unit_test_merge, new_cell_on_existing_cube_and_metric_comprehensive) TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_comprehensive) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int metric_id = fieldstat_register_counter(instance, cube_id, "metric_name"); fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, 5); struct fieldstat *instance_dest = fieldstat_new(); @@ -159,7 +167,7 @@ TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_comprehens TEST(unit_test_merge, new_too_many_cells_on_one_metric_given_source_cube_reset_and_get_different_cube_comprehensive) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2); // limit is 2 + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2); // limit is 2 int metric_id = fieldstat_register_counter(instance, cube_id, "metric name"); fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, 1); struct fieldstat *instance_dest = fieldstat_new(); @@ -187,7 +195,7 @@ TEST(unit_test_merge, new_too_many_cells_on_one_metric_given_source_cube_reset_a TEST(unit_test_merge, new_too_many_cells_on_multiple_metric_given_source_cube_reset_and_get_different_cube_comprehensive) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2); int metric_id1 = fieldstat_register_counter(instance, cube_id, "metric name1"); int metric_id2 = fieldstat_register_counter(instance, cube_id, "metric name2"); fieldstat_counter_incrby(instance, cube_id, metric_id1, &TEST_TAG_STRING, 1, 1); // 1st cell on metric name1 @@ -222,8 +230,9 @@ TEST(unit_test_merge, new_too_many_cells_on_multiple_metric_given_source_cube_re TEST(unit_test_merge, new_cube_and_metric_to_empty_topk) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT, 1, SAMPLING_MODE_TOPK, 10); + fieldstat_cube_create(instance, &TEST_TAG_INT, 1); fieldstat_register_counter(instance, 0, "metric_name"); + fieldstat_cube_set_sampling(instance, 0, SAMPLING_MODE_TOPK, 10, 0); struct fieldstat *instance_dest = fieldstat_new(); @@ -243,8 +252,9 @@ TEST(unit_test_merge, new_cube_and_metric_to_empty_topk) TEST(unit_test_merge, new_cell_on_existing_cube_and_metric_topk) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id = fieldstat_register_counter(instance, cube_id, "metric_name"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, 0); struct fieldstat *instance_dest = fieldstat_new(); fieldstat_merge(instance_dest, instance); @@ -275,8 +285,9 @@ TEST(unit_test_merge, new_cell_on_existing_cube_and_metric_topk) TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_topk) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id = fieldstat_register_counter(instance, cube_id, "metric_name"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, 0); fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, 5); struct fieldstat *instance_dest = fieldstat_new(); @@ -306,8 +317,9 @@ TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_topk) TEST(unit_test_merge, new_too_many_cells_on_one_metric_given_source_cube_reset_and_get_different_cube_topk) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 2); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id = fieldstat_register_counter(instance, cube_id, "metric name"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 2, 0); fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, 1); struct fieldstat *instance_dest = fieldstat_new(); fieldstat_merge(instance_dest, instance); @@ -334,8 +346,9 @@ TEST(unit_test_merge, new_too_many_cells_on_one_metric_given_source_cube_reset_a struct fieldstat *topk_test_push_flows(vector<Fieldstat_tag_list_wrapper *> &flows_in_test, int K, long long count = 1) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, K); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id = fieldstat_register_counter(instance, cube_id, "metric name"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, K, 0); for (size_t i = 0; i < flows_in_test.size(); i++) { fieldstat_counter_incrby(instance, cube_id, metric_id, flows_in_test[i]->get_tag(), flows_in_test[i]->get_tag_count(), count); } @@ -449,9 +462,10 @@ TEST(unit_test_merge, merge_accuracy_test_gen_dest_full_some_inserted_and_some_m TEST(unit_test_merge, primary_metric_has_no_value) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 2); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_primary = fieldstat_register_counter(instance, cube_id, "primary"); int metric_operated = fieldstat_register_counter(instance, cube_id, "operated"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 2, metric_primary); fieldstat_counter_incrby(instance, cube_id, metric_operated, &TEST_TAG_STRING, 1, 1); struct fieldstat *instance_dest = fieldstat_new(); fieldstat_merge(instance_dest, instance); @@ -474,17 +488,19 @@ TEST(unit_test_merge, primary_metric_has_no_value) TEST(unit_test_merge, primary_metric_id_different) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 2); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_primary = fieldstat_register_counter(instance, cube_id, "primary"); int metric_2 = fieldstat_register_counter(instance, cube_id, "2"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 2, metric_primary); + fieldstat_counter_incrby(instance, cube_id, metric_primary, &TEST_TAG_STRING, 1, 100); fieldstat_counter_incrby(instance, cube_id, metric_2, &TEST_TAG_STRING, 1, 1); struct fieldstat *instance_dst = fieldstat_new(); - int cube_id_dst = fieldstat_create_cube(instance_dst, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 2); + int cube_id_dst = fieldstat_cube_create(instance_dst, &TEST_SHARED_TAG, 1); fieldstat_register_counter(instance_dst, cube_id_dst, "2"); int metric_primary_dst = fieldstat_register_counter(instance_dst, cube_id_dst, "primary"); - fieldstat_cube_set_primary_metric(instance_dst, cube_id_dst, metric_primary_dst); + fieldstat_cube_set_sampling(instance_dst, cube_id_dst, SAMPLING_MODE_TOPK, 2, metric_primary_dst); EXPECT_EQ(fieldstat_merge(instance_dst, instance), FS_ERR_INVALID_PARAM); @@ -494,8 +510,9 @@ TEST(unit_test_merge, primary_metric_id_different) TEST(unit_test_merge, new_cube_and_metric_to_empty_spreadsketch) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT, 1, SAMPLING_MODE_TOP_CARDINALITY, 10); + fieldstat_cube_create(instance, &TEST_TAG_INT, 1); fieldstat_register_hll(instance, 0, "metric", 6); + fieldstat_cube_set_sampling(instance, 0, SAMPLING_MODE_TOP_CARDINALITY, 10, 0); struct fieldstat *instance_dest = fieldstat_new(); fieldstat_merge(instance_dest, instance); @@ -513,8 +530,9 @@ TEST(unit_test_merge, new_cube_and_metric_to_empty_spreadsketch) { TEST(unit_test_merge, new_cell_on_existing_cube_and_metric_spreadsketch) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, 10); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id = fieldstat_register_hll(instance, cube_id, "metric", 6); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 10, 0); struct fieldstat *instance_dest = fieldstat_new(); fieldstat_merge(instance_dest, instance); @@ -545,8 +563,9 @@ TEST(unit_test_merge, new_cell_on_existing_cube_and_metric_spreadsketch) { TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_spreadsketch) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, 10); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id = fieldstat_register_hll(instance, cube_id, "metric", 6); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 10, 0); fieldstat_hll_add(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, "1", 1); struct fieldstat *instance_dest = fieldstat_new(); @@ -569,8 +588,9 @@ TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_spreadsket TEST(unit_test_merge, new_too_many_cells_on_one_metric_given_source_cube_reset_and_get_different_cube_spreadsketch) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, 2); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id = fieldstat_register_hll(instance, cube_id, "metric", 6); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 2, 0); fieldstat_hll_add(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, "1", 1); struct fieldstat *instance_dest = fieldstat_new(); fieldstat_merge(instance_dest, instance); @@ -602,8 +622,9 @@ TEST(unit_test_merge, gen_dest_full_all_src_inserted_given_src_flows_larger_spre int K = 100; SpreadSketchZipfGenerator flow_generator(1.0, K); // exactly the number of cells, so there will be almost all(in case of hash collision happen) cells added successfully struct fieldstat *instance_src = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance_src, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, K); + int cube_id = fieldstat_cube_create(instance_src, &TEST_SHARED_TAG, 1); int metric_id = fieldstat_register_hll(instance_src, cube_id, "metric", 6); + fieldstat_cube_set_sampling(instance_src, cube_id, SAMPLING_MODE_TOP_CARDINALITY, K, 0); struct fieldstat *instance_dest = fieldstat_fork(instance_src); const char dest_key[] = "key of dest"; const char src_key[] = "key of src"; @@ -660,8 +681,9 @@ TEST(unit_test_merge, merge_accuracy_test_gen_dest_full_some_inserted_and_some_m int K = 10; SpreadSketchZipfGenerator flow_generator(1.0, K * 10); struct fieldstat *instance_src = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance_src, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, K); + int cube_id = fieldstat_cube_create(instance_src, &TEST_SHARED_TAG, 1); int metric_id = fieldstat_register_hll(instance_src, cube_id, "metric", 6); + fieldstat_cube_set_sampling(instance_src, cube_id, SAMPLING_MODE_TOP_CARDINALITY, K, 0); struct fieldstat *instance_dest = fieldstat_fork(instance_src); std::unordered_map<std::string, std::unordered_set<std::string>> flow_cnt; @@ -699,7 +721,7 @@ TEST(unit_test_merge, merge_accuracy_test_gen_dest_full_some_inserted_and_some_m expected_unique_cnt[kv.first] = kv.second.size(); } double recall = test_cal_topk_accuracy(test_result, expected_unique_cnt); - EXPECT_GT(recall, 0.7); + EXPECT_GE(recall, 0.7); printf("gen_dest_full_all_src_inserted_given_src_flows_larger_spreadsketch recall is %lf\n", recall); fieldstat_free(instance_src); diff --git a/test/test_metric_counter.cpp b/test/test_metric_counter.cpp index d8ce16c..4c30b78 100644 --- a/test/test_metric_counter.cpp +++ b/test/test_metric_counter.cpp @@ -7,15 +7,18 @@ using namespace std; + struct fieldstat *test_init_standard_instance() { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); EXPECT_EQ(cube_id, 0); int metric_id = fieldstat_register_counter(instance, cube_id, "czz_test counter metric"); EXPECT_EQ(metric_id, 0); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_COMPREHENSIVE, 10, 0); + return instance; } @@ -26,6 +29,14 @@ long long my_fieldstat_counter_get(const struct fieldstat *instance, int cube_id return ret; } +int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k, int primary_metric_id=0) +{ + assert(mode == SAMPLING_MODE_COMPREHENSIVE); + int ret = fieldstat_cube_create(instance, dimensions, n_dimensions); + fieldstat_cube_set_sampling(instance, ret, mode, k, primary_metric_id); + return ret; +} + void test_assert_standard_instance(const struct fieldstat *instance) { int *ret_cube_id_arr = NULL; @@ -110,8 +121,9 @@ TEST(metric_test_counter, merge_counter_twice_with_reset) TEST(metric_test_counter, topk_add_and_test_accuracy) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_TOPK, 10); + fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1); fieldstat_register_counter(instance, 0, "test"); + fieldstat_cube_set_sampling(instance, 0, SAMPLING_MODE_TOPK, 10, 0); int tag_list_num = 10000; Fieldstat_tag_list_wrapper *fields[tag_list_num]; @@ -165,7 +177,7 @@ TEST(metric_test_counter, topk_add_and_test_accuracy) TEST(metric_test_counter, add_with_wrong_cube_id_expecting_fail) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_TOPK, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int ret = fieldstat_counter_incrby(instance, cube_id + 1, 0, &TEST_TAG_INT, 1, 1); EXPECT_EQ(ret, FS_ERR_INVALID_CUBE_ID); @@ -178,7 +190,7 @@ TEST(metric_test_counter, add_with_wrong_cube_id_expecting_fail) TEST(metric_test_counter, add_with_wrong_metric_id_expecting_fail) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_TOPK, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int metric_id = fieldstat_register_counter(instance, 0, "test"); int ret = fieldstat_counter_incrby(instance, cube_id, metric_id + 1, &TEST_TAG_INT, 1, 1); @@ -192,9 +204,10 @@ TEST(metric_test_counter, add_with_wrong_metric_id_expecting_fail) TEST(metric_test_counter, add_and_query_on_dummy_cell_of_topk) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); fieldstat_register_counter(instance, cube_id, "primary"); // also the dummy one int metric_id = fieldstat_register_counter(instance, cube_id, "using"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, 0); fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1); @@ -208,11 +221,10 @@ TEST(metric_test_counter, add_and_query_on_dummy_cell_of_topk) TEST(metric_test_counter, primary_counter_add_after_first) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id_primary = fieldstat_register_counter(instance, cube_id, "primary"); int metric_id2 = fieldstat_register_counter(instance, cube_id, "using"); - fieldstat_cube_set_primary_metric(instance, cube_id, metric_id_primary); - + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, metric_id_primary); int ret = fieldstat_counter_incrby(instance, cube_id, metric_id2, &TEST_TAG_INT, 1, 10); EXPECT_EQ(ret, FS_OK); diff --git a/test/test_metric_histogram.cpp b/test/test_metric_histogram.cpp index 3a28aa3..3ef171c 100644 --- a/test/test_metric_histogram.cpp +++ b/test/test_metric_histogram.cpp @@ -6,10 +6,18 @@ #include "hdr/hdr_histogram.h" #include "histogram_encoder.h" +int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k) +{ + assert(mode == SAMPLING_MODE_COMPREHENSIVE); + int ret = fieldstat_cube_create(instance, dimensions, n_dimensions); + fieldstat_cube_set_sampling(instance, ret, mode, k, 0); + return ret; +} + struct fieldstat *test_init_standard_instance_one_cube_one_metric_one_cell_hdr() { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); EXPECT_EQ(cube_id, 0); int metric_id = fieldstat_register_histogram(instance, cube_id, "czz_test hdr metric", 1, 600000, 3); @@ -107,7 +115,7 @@ TEST(metric_test_histogram, merge_twice_with_reset) TEST(metric_test_histogram, add_with_wrong_cube_id_expecting_fail) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int ret = fieldstat_histogram_record(instance, cube_id + 1, 0, &TEST_TAG_INT, 1, 1); EXPECT_EQ(ret, FS_ERR_INVALID_CUBE_ID); @@ -120,7 +128,7 @@ TEST(metric_test_histogram, add_with_wrong_cube_id_expecting_fail) TEST(metric_test_histogram, add_with_wrong_metric_id_expecting_fail) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int metric_id = fieldstat_register_histogram(instance, cube_id, "czz_test", 1, 600000, 3); int ret = fieldstat_histogram_record(instance, cube_id, metric_id + 1, &TEST_TAG_INT, 1, 1); @@ -169,7 +177,7 @@ TEST(metric_test_histogram, encode_decode_b64) TEST(metric_test_histogram, can_add_0value) // histogram only allow min_val > 0, but it can accept value == 0 { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int metric_id = fieldstat_register_histogram(instance, cube_id, "czz_test", 1, 600000, 3); int ret = fieldstat_histogram_record(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 0); diff --git a/test/test_metric_hll.cpp b/test/test_metric_hll.cpp index 204c762..713c00d 100644 --- a/test/test_metric_hll.cpp +++ b/test/test_metric_hll.cpp @@ -8,10 +8,18 @@ #include "fieldstat.h" #include "utils.hpp" +int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k) +{ + assert(mode == SAMPLING_MODE_COMPREHENSIVE); + int ret = fieldstat_cube_create(instance, dimensions, n_dimensions); + fieldstat_cube_set_sampling(instance, ret, mode, k, 0); + return ret; +} + struct fieldstat *test_init_standard_instance_one_cube_one_metric_one_cell_hll(bool is_gauge = false) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); EXPECT_EQ(cube_id, 0); int metric_id = fieldstat_register_hll(instance, cube_id, "czz_test hll metric", 10); @@ -198,7 +206,7 @@ TEST(metric_test_hll, serialize_with_b64_and_query_with_python_api) TEST(metric_test_hll, add_with_wrong_cube_id_expecting_fail) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int ret = fieldstat_hll_add(instance, cube_id + 1, 0, &TEST_TAG_INT, 1, "hello", 5); EXPECT_EQ(ret, FS_ERR_INVALID_CUBE_ID); @@ -211,7 +219,7 @@ TEST(metric_test_hll, add_with_wrong_cube_id_expecting_fail) TEST(metric_test_hll, add_with_wrong_metric_id_expecting_fail) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int metric_id = fieldstat_register_hll(instance, cube_id, "czz_test hll metric", 10); int ret = fieldstat_hll_add(instance, cube_id, metric_id + 1, &TEST_TAG_INT, 1, "hello", 5); @@ -226,8 +234,9 @@ TEST(metric_test_hll, spread_sketch_add_and_test_accuracy) { struct fieldstat *instance = fieldstat_new(); int K = 10; - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_TOP_CARDINALITY, K); + fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1); fieldstat_register_hll(instance, 0, "testss", 6); + fieldstat_cube_set_sampling(instance, 0, SAMPLING_MODE_TOP_CARDINALITY, K, 0); int n_flows = 100000; std::unordered_map<std::string, std::unordered_set<std::string>> flow_cnt; diff --git a/test/test_performance.cpp b/test/test_performance.cpp index b1ada45..d0ca633 100644 --- a/test/test_performance.cpp +++ b/test/test_performance.cpp @@ -6,6 +6,15 @@ #include "fieldstat_exporter.h" #include "utils.hpp" + +int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *tag, size_t tag_count, enum sampling_mode mode, int k, int primary_metric_id=0) +{ + int ret = fieldstat_cube_create(instance, tag, tag_count); + fieldstat_cube_set_sampling(instance, ret, mode, k, primary_metric_id); + return ret; +} + + // /* -------------------------------------------------------------------------- */ // /* merge */ // /* -------------------------------------------------------------------------- */ @@ -18,8 +27,9 @@ double perform_merge_test(std::function<void (struct fieldstat*, int, int, const fields[i] = new Fieldstat_tag_list_wrapper("my key", i); } struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, mode, MAX_CELL_NUM); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id = metric_register_func(instance); + fieldstat_cube_set_sampling(instance, cube_id, mode, MAX_CELL_NUM, metric_id); for (int j = 0; j < MAX_CELL_NUM; j++) { metric_add_func(instance, cube_id, metric_id, fields[j]->get_tag(), 1); } @@ -163,11 +173,13 @@ struct fieldstat *construct_fs_with_many_empty_cubes(int cube_num, int metric_nu struct field tmp_tag = TEST_TAG_INT; for (int i = 0; i < cube_num; i++) { tmp_tag.value_longlong = i; - int cube_id = fieldstat_create_cube(instance, &tmp_tag, 1, mode, 1000); + int cube_id = fieldstat_cube_create(instance, &tmp_tag, 1); for (int j = 0; j < metric_num; j++) { fieldstat_register_counter(instance, cube_id, std::to_string(j).c_str()); } + + fieldstat_cube_set_sampling(instance, cube_id, mode, 1000, 0); } return instance; } @@ -219,7 +231,7 @@ TEST(test_performance, performance_test_add_cells_comprehensive) } // getchar(); struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, cell_count); + test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, cell_count); fieldstat_register_counter(instance, 0, "test"); clock_t start = clock(); @@ -246,8 +258,9 @@ TEST(test_performance, performance_test_add_cells_topk) fields[i].value_longlong = rand() % 1000; } struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_TOPK, 1000); + fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1); fieldstat_register_counter(instance, 0, "test"); + fieldstat_cube_set_sampling(instance, 0, SAMPLING_MODE_TOPK, 1000, 0); // getchar(); clock_t start = clock(); @@ -267,7 +280,7 @@ TEST(test_performance, performance_test_add_cells_topk) TEST(test_performance, performance_test_add_cells_histogram_record) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); fieldstat_register_histogram(instance, 0, "test", 1, 100000, 3); size_t test_num = 100000; long long vals[test_num]; @@ -288,7 +301,7 @@ TEST(test_performance, performance_test_add_cells_histogram_record) TEST(test_performance, performance_test_add_cells_hll_add) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); fieldstat_register_hll(instance, 0, "test", 6); size_t test_num = 100000; std::string vals[test_num]; @@ -325,7 +338,7 @@ TEST(test_performance, performance_test_add_cells_comprehensive_5_tags) } // getchar(); struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, cell_count); + test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, cell_count); fieldstat_register_counter(instance, 0, "test"); clock_t start = clock(); @@ -346,7 +359,7 @@ TEST(test_performance, performance_test_add_cells_comprehensive_5_tags) TEST(test_performance, performance_test_add_cells_histogram_record_5tags) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); fieldstat_register_histogram(instance, 0, "test", 1, 100000, 3); size_t test_num = 100000; long long vals[test_num]; @@ -374,7 +387,7 @@ TEST(test_performance, performance_test_add_cells_histogram_record_5tags) TEST(test_performance, performance_test_add_cells_hll_add_5tags) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); fieldstat_register_hll(instance, 0, "test", 6); size_t test_num = 100000; std::string vals[test_num]; @@ -419,7 +432,7 @@ TEST(test_performance, export_many_cells) struct fieldstat *instance = fieldstat_new(); for (int i = 0; i < CUBE_NUM; i++) { Fieldstat_tag_list_wrapper cube_tag("shared key", i); - int cube_id = fieldstat_create_cube(instance, cube_tag.get_tag(), cube_tag.get_tag_count(), SAMPLING_MODE_COMPREHENSIVE, MAX_CELL_NUM); + int cube_id = test_fieldstat_cube_create(instance, cube_tag.get_tag(), cube_tag.get_tag_count(), SAMPLING_MODE_COMPREHENSIVE, MAX_CELL_NUM); for (int j = 0; j < METRIC_NUM; j++) { string metric_name = "metric name" + to_string(i) + to_string(j); @@ -493,10 +506,10 @@ TEST(test_performance, callibrate_unchanged) fieldstat_free(instance_dest); } -struct fieldstat *construct_fs_with_many_cells(int cell_num, enum sampling_mode mode) +struct fieldstat *construct_fs_with_many_cells(int cell_num) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT, 1, mode, cell_num); + test_fieldstat_cube_create(instance, &TEST_TAG_INT, 1, SAMPLING_MODE_COMPREHENSIVE, cell_num); fieldstat_register_counter(instance, 0, "test"); struct field tmp_tag = TEST_TAG_INT; for (int i = 0; i < cell_num; i++) { @@ -509,7 +522,7 @@ struct fieldstat *construct_fs_with_many_cells(int cell_num, enum sampling_mode TEST(test_performance, reset_many_cells) { - struct fieldstat *instance = construct_fs_with_many_cells(1000, SAMPLING_MODE_COMPREHENSIVE); // many empty cubes, 10 metrics is a common case + struct fieldstat *instance = construct_fs_with_many_cells(1000); // many empty cubes, 10 metrics is a common case clock_t start = clock(); fieldstat_reset(instance); clock_t end = clock(); diff --git a/test/test_register_and_reset.cpp b/test/test_register_and_reset.cpp index 31f57ba..380b156 100644 --- a/test/test_register_and_reset.cpp +++ b/test/test_register_and_reset.cpp @@ -3,13 +3,20 @@ #include "fieldstat.h" #include "utils.hpp" +int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k) +{ + assert(mode == SAMPLING_MODE_COMPREHENSIVE); + int ret = fieldstat_cube_create(instance, dimensions, n_dimensions); + fieldstat_cube_set_sampling(instance, ret, mode, k, 0); + return ret; +} TEST(test_register, delete_cube_and_register_and_origin_position) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - fieldstat_destroy_cube(instance, cube_id); - cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + fieldstat_cube_destroy(instance, cube_id); + cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); EXPECT_EQ(cube_id, 0); EXPECT_EQ(fieldstat_find_cube(instance, &TEST_SHARED_TAG, 1) , 0); fieldstat_free(instance); @@ -18,11 +25,11 @@ TEST(test_register, delete_cube_and_register_and_origin_position) TEST(test_register, delete_comprehensive_cube_with_cells_and_metrics) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1); - fieldstat_destroy_cube(instance, cube_id); + fieldstat_cube_destroy(instance, cube_id); struct field_list *tag_list = fieldstat_cube_get_tags(instance, cube_id); EXPECT_EQ(tag_list, nullptr); @@ -35,11 +42,12 @@ TEST(test_register, delete_comprehensive_cube_with_cells_and_metrics) TEST(test_register, delete_topk_cube_with_cells_and_metrics) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, metric_id); fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1); - fieldstat_destroy_cube(instance, cube_id); + fieldstat_cube_destroy(instance, cube_id); struct field_list *tag_list = fieldstat_cube_get_tags(instance, cube_id); EXPECT_EQ(tag_list, nullptr); int cube_id_ret = fieldstat_find_cube(instance, &TEST_SHARED_TAG, 1); @@ -51,14 +59,15 @@ TEST(test_register, delete_topk_cube_with_cells_and_metrics) TEST(test_register, delete_spreadsketch_cube_with_cells_and_metrics) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, 10); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id1 = fieldstat_register_counter(instance, cube_id, "counter"); int metric_primary = fieldstat_register_hll(instance, cube_id, "hll_primary", 5); - fieldstat_cube_set_primary_metric(instance, cube_id, metric_primary); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 10, metric_primary); + fieldstat_counter_incrby(instance, cube_id, metric_id1, &TEST_TAG_INT, 1, 1); fieldstat_hll_add_field(instance, cube_id, metric_primary, &TEST_TAG_INT, 1, &TEST_TAG_DOUBLE, 1); - fieldstat_destroy_cube(instance, cube_id); + fieldstat_cube_destroy(instance, cube_id); struct field_list *tag_list = fieldstat_cube_get_tags(instance, cube_id); EXPECT_EQ(tag_list, nullptr); int cube_id_ret = fieldstat_find_cube(instance, &TEST_SHARED_TAG, 1); @@ -67,7 +76,6 @@ TEST(test_register, delete_spreadsketch_cube_with_cells_and_metrics) fieldstat_free(instance); } - int test_get_max_metric_id(const struct fieldstat *instance) { int *metric_id_out; @@ -80,7 +88,7 @@ int test_get_max_metric_id(const struct fieldstat *instance) TEST(test_register, reset_and_try_to_query_cell_comprehensive) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1); @@ -99,8 +107,10 @@ TEST(test_register, reset_and_try_to_query_cell_comprehensive) TEST(test_register, reset_and_try_to_query_cell_topk) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, metric_id); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1); fieldstat_reset(instance); @@ -118,8 +128,9 @@ TEST(test_register, reset_and_try_to_query_cell_topk) TEST(test_register, reset_and_try_to_query_cell_spreadsketch) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, 10); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id = fieldstat_register_hll(instance, cube_id, "hll", 5); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 10, metric_id); fieldstat_hll_add(instance, cube_id, metric_id, &TEST_TAG_INT, 1, "12abc", 5); fieldstat_reset(instance); @@ -137,7 +148,7 @@ TEST(test_register, reset_and_try_to_query_cell_spreadsketch) TEST(test_register, reset_and_new_cell_comprehensive) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2); int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1); fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_DOUBLE, 1, 1); @@ -154,8 +165,9 @@ TEST(test_register, reset_and_new_cell_comprehensive) TEST(test_register, reset_and_new_cell_topk) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 1); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 1, metric_id); fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 100);//100: bigger value int ret = fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_DOUBLE, 1, 1); EXPECT_EQ(ret, FS_ERR_TOO_MANY_CELLS); @@ -170,8 +182,9 @@ TEST(test_register, reset_and_new_cell_topk) TEST(test_register, reset_and_new_cell_spreadsketch) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, 1); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id = fieldstat_register_hll(instance, cube_id, "hll", 5); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 1, metric_id); // spread sketch will store more data than expected cell number 1. So loop for many cells first to trigger the error struct field test_tag_long = TEST_TAG_INT; for (int i = 0; i < 10000; i++) { @@ -191,7 +204,7 @@ TEST(test_register, reset_and_new_cell_spreadsketch) TEST(test_register, ensure_recovery_more_faster_comprehensive) { struct fieldstat *instance = fieldstat_new(); int cell_num = 1000; - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, cell_num); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, cell_num); int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); struct field test_tag_long = TEST_TAG_INT; @@ -221,8 +234,10 @@ TEST(test_register, ensure_recovery_more_faster_comprehensive) { TEST(test_register, ensure_recovery_more_faster_topk) { struct fieldstat *instance = fieldstat_new(); int cell_num = 1000; - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, cell_num); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, cell_num, metric_id); + struct field test_tag_long = TEST_TAG_INT; clock_t start = clock(); @@ -251,8 +266,9 @@ TEST(test_register, ensure_recovery_more_faster_topk) { TEST(test_register, ensure_recovery_more_faster_spreadsketch) { struct fieldstat *instance = fieldstat_new(); int cell_num = 1000; - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, cell_num); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id = fieldstat_register_hll(instance, cube_id, "counter", 6); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, cell_num, metric_id); struct field test_tag_long = TEST_TAG_INT; clock_t start = clock(); @@ -285,7 +301,7 @@ TEST(test_register, register_many_cubes) struct field shared_tag = TEST_SHARED_TAG; for (int i = 0; i < registered_cube; i++) { shared_tag.value_longlong = i; - int cube_id = fieldstat_create_cube(instance, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10); EXPECT_EQ(cube_id, i); fieldstat_register_counter(instance, cube_id, "counter"); } @@ -306,7 +322,7 @@ TEST(test_register, register_many_cubes) TEST(test_register, add_many_tagged_cells) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); size_t n_field = 1000; struct field test_tag_long[n_field]; for (size_t i = 0; i < n_field; i++) { @@ -327,7 +343,7 @@ TEST(test_register, add_many_tagged_cells) TEST(test_register, add_long_tagged_cells) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); struct field test_tag_long = TEST_TAG_STRING; char *long_string = (char *)malloc(5001); memset(long_string, 'a', 5001); @@ -349,7 +365,7 @@ TEST(test_register, add_long_tagged_cells) TEST(test_register, register_many_metrics) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int metric_id = 0; for (int i = 0; i < 200; i++) { metric_id = fieldstat_register_counter(instance, cube_id, (std::string("counter ") + std::to_string(i)).c_str()); @@ -381,13 +397,13 @@ TEST(test_register, fork_registered_info_from_empty_one) TEST(test_register, fork_registered_info_with_cube_and_metric) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); int metric_id2 = fieldstat_register_counter(instance, cube_id, "counter2"); fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1); - int cube_id_del = fieldstat_create_cube(instance, &TEST_FIELD_VALUE_DOUBLE_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - fieldstat_create_cube(instance, &TEST_TAG_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - fieldstat_destroy_cube(instance, cube_id_del); + int cube_id_del = test_fieldstat_cube_create(instance, &TEST_FIELD_VALUE_DOUBLE_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + test_fieldstat_cube_create(instance, &TEST_TAG_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + fieldstat_cube_destroy(instance, cube_id_del); struct fieldstat *dup = fieldstat_fork(instance); @@ -423,8 +439,8 @@ TEST(test_register, unregister_cube_on_wrong_instance) struct fieldstat *instance = fieldstat_new(); struct fieldstat *instance_dst = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int cube_id2 = fieldstat_create_cube(instance, &TEST_TAG_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id2 = test_fieldstat_cube_create(instance, &TEST_TAG_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); int metric_id2 = fieldstat_register_counter(instance, cube_id2, "counter2"); fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1); @@ -432,7 +448,7 @@ TEST(test_register, unregister_cube_on_wrong_instance) fieldstat_merge(instance_dst, instance); - fieldstat_destroy_cube(instance_dst, 0); + fieldstat_cube_destroy(instance_dst, 0); int *cube_ids; int cube_num; @@ -457,7 +473,7 @@ TEST(test_register, unregister_cube_on_wrong_instance) TEST(test_register, register_many_cells_on_unlimited_sized_cube) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 0); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 0); int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); struct field test_tag = {"abc", FIELD_VALUE_INTEGER, {.value_longlong = 0}}; for (int i = 0; i < 10000; i++) { @@ -478,15 +494,15 @@ TEST(test_register, register_many_cells_on_unlimited_sized_cube) TEST(test_register, register_cube_twice) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); - int cube_id2 = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); + test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id2 = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); EXPECT_EQ(cube_id2, FS_ERR_INVALID_KEY); fieldstat_free(instance); } TEST(test_register, find_cube) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int find_cube_id = fieldstat_find_cube(instance, &TEST_SHARED_TAG, 1); EXPECT_EQ(cube_id, find_cube_id); @@ -498,7 +514,7 @@ TEST(test_register, find_cube) { TEST(test_register, register_metric_twice) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); fieldstat_register_counter(instance, cube_id, "counter"); int metric_id2 = fieldstat_register_counter(instance, cube_id, "counter"); @@ -506,10 +522,20 @@ TEST(test_register, register_metric_twice) { fieldstat_free(instance); } +TEST(test_register, add_on_uninitialized_cube) { + struct fieldstat *instance = fieldstat_new(); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + // fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1); + EXPECT_DEATH(fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1), ".*"); + + fieldstat_free(instance); +} + TEST(calibrate, target_one_more_metric) { struct fieldstat *master = fieldstat_new(); - int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); fieldstat_register_counter(master, cube_id, "counter"); struct fieldstat *target = fieldstat_fork(master); EXPECT_EQ(fieldstat_register_counter(target, cube_id, "counter2"), 1); @@ -529,7 +555,7 @@ TEST(calibrate, target_one_more_metric) TEST(calibrate, master_one_more_metric) { struct fieldstat *master = fieldstat_new(); - int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); fieldstat_register_counter(master, cube_id, "counter"); struct fieldstat *target = fieldstat_fork(master); EXPECT_EQ(fieldstat_register_counter(master, cube_id, "counter2"), 1); @@ -551,7 +577,7 @@ TEST(calibrate, master_one_more_metric) TEST(calibrate, different_metric) { struct fieldstat *master = fieldstat_new(); - int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); fieldstat_register_counter(master, cube_id, "counter"); struct fieldstat *target = fieldstat_fork(master); EXPECT_EQ(fieldstat_register_counter(target, cube_id, "counter2"), 1); @@ -573,10 +599,10 @@ TEST(calibrate, different_metric) TEST(calibrate, target_more_cube) { struct fieldstat *master = fieldstat_new(); - int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); fieldstat_register_counter(master, cube_id, "counter"); struct fieldstat *target = fieldstat_fork(master); - int cube_id2 = fieldstat_create_cube(target, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id2 = test_fieldstat_cube_create(target, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10); EXPECT_EQ(cube_id2, 1); fieldstat_register_counter(target, cube_id, "counter2"); @@ -605,9 +631,9 @@ TEST(calibrate, target_more_cube) TEST(calibrate, master_more_cube) { struct fieldstat *master = fieldstat_new(); - int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); struct fieldstat *target = fieldstat_fork(master); - int cube_id2 = fieldstat_create_cube(master, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id2 = test_fieldstat_cube_create(master, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10); fieldstat_calibrate(master, target); @@ -636,10 +662,10 @@ TEST(calibrate, master_more_cube) TEST(calibrate, master_change_cube) { struct fieldstat *master = fieldstat_new(); - int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); struct fieldstat *target = fieldstat_fork(master); - fieldstat_destroy_cube(master, cube_id); - fieldstat_create_cube(master, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + fieldstat_cube_destroy(master, cube_id); + test_fieldstat_cube_create(master, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10); fieldstat_calibrate(master, target); @@ -660,14 +686,14 @@ TEST(calibrate, master_many_cube) struct field shared_tag = TEST_SHARED_TAG; for (int i = 0; i < registered_cube; i++) { shared_tag.value_longlong = i; - int cube_id = fieldstat_create_cube(master, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(master, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10); EXPECT_EQ(cube_id, i); } struct fieldstat *target = fieldstat_fork(master); int new_registered = 5000; for (int i = 0; i < new_registered; i++) { shared_tag.value_longlong = i + registered_cube; - int cube_id = fieldstat_create_cube(master, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(master, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10); EXPECT_EQ(cube_id, i + registered_cube); } @@ -707,13 +733,13 @@ TEST(calibrate, issue_calibrate_wrong) struct fieldstat *master = fieldstat_new(); const struct field *tag_A = &TEST_SHARED_TAG; const struct field *tag_B = &TEST_TAG_INT; - int cube_idA = fieldstat_create_cube(master, tag_A, 1, SAMPLING_MODE_COMPREHENSIVE, 1); - int cube_idB = fieldstat_create_cube(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1); + int cube_idA = test_fieldstat_cube_create(master, tag_A, 1, SAMPLING_MODE_COMPREHENSIVE, 1); + int cube_idB = test_fieldstat_cube_create(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1); struct fieldstat *target = fieldstat_fork(master); - EXPECT_EQ(fieldstat_destroy_cube(master, cube_idA), FS_OK); - EXPECT_EQ(fieldstat_destroy_cube(master, cube_idB), FS_OK); - int cube_idBa = fieldstat_create_cube(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1); + EXPECT_EQ(fieldstat_cube_destroy(master, cube_idA), FS_OK); + EXPECT_EQ(fieldstat_cube_destroy(master, cube_idB), FS_OK); + int cube_idBa = test_fieldstat_cube_create(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1); EXPECT_EQ(cube_idBa, 0); fieldstat_calibrate(master, target); @@ -739,11 +765,11 @@ TEST(calibrate, delete_first_cube) struct fieldstat *master = fieldstat_new(); const struct field *tag_A = &TEST_SHARED_TAG; const struct field *tag_B = &TEST_TAG_INT; - int cube_idA = fieldstat_create_cube(master, tag_A, 1, SAMPLING_MODE_COMPREHENSIVE, 1); - int cube_idB = fieldstat_create_cube(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1); + int cube_idA = test_fieldstat_cube_create(master, tag_A, 1, SAMPLING_MODE_COMPREHENSIVE, 1); + int cube_idB = test_fieldstat_cube_create(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1); struct fieldstat *target = fieldstat_fork(master); - fieldstat_destroy_cube(master, cube_idA); + fieldstat_cube_destroy(master, cube_idA); fieldstat_calibrate(master, target); diff --git a/test/test_write_json_file.cpp b/test/test_write_json_file.cpp index ff3dedf..1dba94b 100644 --- a/test/test_write_json_file.cpp +++ b/test/test_write_json_file.cpp @@ -58,6 +58,14 @@ extern "C" { return fse; } +int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k) +{ + assert(mode == SAMPLING_MODE_COMPREHENSIVE); + int ret = fieldstat_cube_create(instance, dimensions, n_dimensions); + fieldstat_cube_set_sampling(instance, ret, mode, k, 0); + return ret; +} + static void write_hll(struct fieldstat *instance) { struct field shared_tags[1]; @@ -66,7 +74,7 @@ static void write_hll(struct fieldstat *instance) { shared_tags[0].value_longlong = 1; const char *hll_name[] = {"external_ip", "internal_ip", "acc_ip"}; - int cube_id = fieldstat_create_cube(instance, shared_tags, 1, + int cube_id = test_fieldstat_cube_create(instance, shared_tags, 1, SAMPLING_MODE_COMPREHENSIVE, 100); for(unsigned int i = 0; i < sizeof(hll_name) / sizeof(hll_name[0]); i++) @@ -107,7 +115,7 @@ void write_histogram(struct fieldstat *instance) { "bye/udp", "oth_mtd/udp"}; - int cube_id = fieldstat_create_cube(instance, shared_tags, 2, + int cube_id = test_fieldstat_cube_create(instance, shared_tags, 2, SAMPLING_MODE_COMPREHENSIVE, 100); for(unsigned int i = 0; i < sizeof(hist_names)/sizeof(hist_names[0]); i++) @@ -185,7 +193,7 @@ void write_table(struct fieldstat *instance) { cell_tags.type = FIELD_VALUE_CSTRING; cell_tags.value_str = "true"; - int cube_id = fieldstat_create_cube(instance, shared_tags, 2, + int cube_id = test_fieldstat_cube_create(instance, shared_tags, 2, SAMPLING_MODE_COMPREHENSIVE, 100); int counter_id_0 = fieldstat_register_counter(instance, cube_id, "T_success_log"); |
