summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorchenzizhan <[email protected]>2024-07-17 15:46:47 +0800
committerchenzizhan <[email protected]>2024-07-17 15:46:47 +0800
commit8ce45ff4f96186c5d9e0f3e82addd2085d7c8788 (patch)
treeccd28d85c7024d2dd4474ae8e6b873adca74b5eb
parentdccb4ce1fd92b1f142383e585487af08831264d3 (diff)
cube set sampling
-rw-r--r--include/fieldstat/fieldstat.h13
-rw-r--r--readme_fieldstat.md2
-rw-r--r--src/cells/hash_table.c3
-rw-r--r--src/cells/heavy_keeper.c3
-rw-r--r--src/cells/spread_sketch.c3
-rw-r--r--src/cube.c86
-rw-r--r--src/cube.h3
-rw-r--r--src/fieldstat.c46
-rw-r--r--src/fieldstat_easy.c3
-rw-r--r--test/profiling/main.c2
-rw-r--r--test/test_empty_tags.cpp13
-rw-r--r--test/test_exporter_json.cpp57
-rw-r--r--test/test_fuzz_test.cpp37
-rw-r--r--test/test_merge.cpp70
-rw-r--r--test/test_metric_counter.cpp28
-rw-r--r--test/test_metric_histogram.cpp16
-rw-r--r--test/test_metric_hll.cpp17
-rw-r--r--test/test_performance.cpp39
-rw-r--r--test/test_register_and_reset.cpp138
-rw-r--r--test/test_write_json_file.cpp14
20 files changed, 383 insertions, 210 deletions
diff --git a/include/fieldstat/fieldstat.h b/include/fieldstat/fieldstat.h
index b5fef15..0f77ec4 100644
--- a/include/fieldstat/fieldstat.h
+++ b/include/fieldstat/fieldstat.h
@@ -67,11 +67,8 @@ int fieldstat_calibrate(const struct fieldstat *master, struct fieldstat *replic
* @param max_n_cell: max number of samplings(cells) in each cube. When mode is TOPK, max_n_cell > 0, while in COMPREHENSIVE mode, max_n_cell can be 0, meaning that there is no limit.
* @return cube id, if success; otherwise, return FS_ERR_NULL_HANDLER, or FS_ERR_INVALID_PARAM when (max_n_cell == 0 && mode == TOPK). return FS_ERR_INVALID_KEY when the cube_dimensions is not unique.
*/
-int fieldstat_create_cube(struct fieldstat *instance, const struct field *cube_dimensions, size_t n_dimension, enum sampling_mode mode, size_t max_n_cell);
-// todo: 重命名为fieldstat_cube_create
-
-//todo: create cube 接口变化
-// int fieldstat_cube_set_sampling(struct fieldstat *instance, int cube_id, enum sampling_mode mode, int max_n_cell, int primary_metric_id);
+int fieldstat_cube_create(struct fieldstat *instance, const struct field *cube_dimensions, size_t n_dimension);
+int fieldstat_cube_set_sampling(struct fieldstat *instance, int cube_id, enum sampling_mode mode, int max_n_cell, int primary_metric_id);
/*
@brief Change the topk cube primary metric id. When fieldstat_counter_add or fieldstat_counter_set are called on the primary metric, the topk record of such cell will be updated.
@@ -87,7 +84,7 @@ int fieldstat_cube_set_primary_metric(struct fieldstat *instance, int cube_id, i
* @brief Delete the cube of cube_id. All the cells and metrics are deleted. The cube_id may be reused by other new cubes. Increase the corresponding cube_version by 1.
* @return FS_OK, FS_ERR_NULL_HANDLER or FS_ERR_INVALID_CUBE_ID
*/
-int fieldstat_destroy_cube(struct fieldstat *instance, int cube_id);
+int fieldstat_cube_destroy(struct fieldstat *instance, int cube_id);
/*
* @brief add a metric to the cube of cube_id. One metric may be associated with different cells.
@@ -114,7 +111,7 @@ int fieldstat_register_histogram(struct fieldstat *instance, int cube_id, const
/*
* @brief let the value of counter metric of cell_id increase by `increment`.
- * @param cube_id: cube id, previously returned by fieldstat_create_cube.
+ * @param cube_id: cube id, previously returned by fieldstat_cube_create.
* @param metric_id: metric id, previously returned by fieldstat_register_counter.
* @param increment: increment of the counter metric. Can be negative.
* @return FS_OK if success. FS_ERR_NULL_HANDLER, FS_ERR_INVALID_CUBE_ID, FS_ERR_INVALID_METRIC_ID if fail.
@@ -200,7 +197,7 @@ enum metric_type fieldstat_get_metric_type(const struct fieldstat *instance, int
void fieldstat_cube_get_cells(const struct fieldstat *instance, int cube_id, struct field_list **cell_dimensions, size_t *n_cell);
/*
- get the field of fieldstat_create_cube. User free them by calling fieldstat_tag_list_arr_free(struct field_list *, 1)
+ get the field of fieldstat_cube_create. User free them by calling fieldstat_tag_list_arr_free(struct field_list *, 1)
return NULL when ID is invalid.
*/
struct field_list *fieldstat_cube_get_tags(const struct fieldstat *instance, int cube_id);
diff --git a/readme_fieldstat.md b/readme_fieldstat.md
index 42c6680..4dd7a99 100644
--- a/readme_fieldstat.md
+++ b/readme_fieldstat.md
@@ -40,7 +40,7 @@ Download fieldstat4 rpm from https://repo.geedge.net/pulp/content/ and install r
#include "fieldstat.h"
struct fieldstat *instance = fieldstat_new();
-int cube_id = fieldstat_create_cube(instance, YOUR_SHARED_TAG, YOUR_SHARED_TAG_LENGTH, SAMPLING_MODE_TOPK, MAX_CELL_NUMBER);
+int cube_id = fieldstat_cube_create(instance, YOUR_SHARED_TAG, YOUR_SHARED_TAG_LENGTH, SAMPLING_MODE_TOPK, MAX_CELL_NUMBER);
int metric_counter_id = fieldstat_register_counter(instance, cube_id, "any metric name", 0/1);
int metric_histogram_id = fieldstat_register_histogramogram(instance, cube_id, "any metric name", THE_MINIMUM_NUMBER_TO_RECORD, THE_MAXIMUM_NUMBER_TO_RECORD, PRECISION);
int metric_hll_id = fieldstat_register_hll(instance, cube_id, "any metric name", PRECISION);
diff --git a/src/cells/hash_table.c b/src/cells/hash_table.c
index d6dbfb5..60d2465 100644
--- a/src/cells/hash_table.c
+++ b/src/cells/hash_table.c
@@ -64,6 +64,9 @@ struct hash_table *hash_table_new(int max_query_num) {
}
void hash_table_free(struct hash_table *pthis) {
+ if (pthis == NULL) {
+ return;
+ }
struct tag_exdata_item *item, *tmp;
HASH_ITER(hh, pthis->tag_id_map, item, tmp) {
HASH_DEL(pthis->tag_id_map, item);
diff --git a/src/cells/heavy_keeper.c b/src/cells/heavy_keeper.c
index b606a67..c62e550 100644
--- a/src/cells/heavy_keeper.c
+++ b/src/cells/heavy_keeper.c
@@ -518,6 +518,9 @@ struct heavy_keeper *heavy_keeper_new(int max_query_num) {
}
void heavy_keeper_free(struct heavy_keeper *hk) {
+ if (hk == NULL) {
+ return;
+ }
sorted_set_free(hk->top_K_heap);
free(hk->sketch);
diff --git a/src/cells/spread_sketch.c b/src/cells/spread_sketch.c
index 5d5dd72..3e9b11e 100644
--- a/src/cells/spread_sketch.c
+++ b/src/cells/spread_sketch.c
@@ -382,6 +382,9 @@ struct spread_sketch *duplicate_and_step(const struct spread_sketch *ss, const s
}
void spread_sketch_free(struct spread_sketch *ss) {
+ if (ss == NULL) {
+ return;
+ }
smart_ptr_table_free(ss->table);
for (int i = 0; i < ss->depth * ss->width; i++) {
hll_free_register(ss->buckets[i].sthll_register);
diff --git a/src/cube.c b/src/cube.c
index 4bd3305..abc3a66 100644
--- a/src/cube.c
+++ b/src/cube.c
@@ -497,10 +497,9 @@ void *exdata_copy_i(void *exdata) {
return cell_copy((struct cell *)exdata);
}
-struct cube *cube_info_new(const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, size_t max_n_cell)
+struct cube *cube_info_new(const struct field *dimensions, size_t n_dimensions)
{
struct cube *cube = calloc(1, sizeof(struct cube));
- cube->sampling_mode = mode;
if (n_dimensions == 0) {
cube->cube_dimensions = NULL;
@@ -509,42 +508,81 @@ struct cube *cube_info_new(const struct field *dimensions, size_t n_dimensions,
}
cube->n_dimensions = n_dimensions;
- cube->max_n_cell = max_n_cell;
field_array_to_key(dimensions, n_dimensions, &cube->serialized_dimensions, &cube->serialized_dimensions_len);
cube->id = -1;
+ cube->primary_metric_id = -1;
return cube;
}
-struct cube *cube_new(const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, size_t max_n_cell)
+struct cube *cube_new(const struct field *dimensions, size_t n_dimensions)
{
- struct cube *cube = cube_info_new(dimensions, n_dimensions, mode, max_n_cell);
+ struct cube *cube = cube_info_new(dimensions, n_dimensions);
cube->manifest_manager = metric_manifest_manager_new();
- switch (mode)
- {
+ return cube;
+}
+
+int cube_set_sampling_mode(struct cube *cube, enum sampling_mode mode, int max_n_cell, int primary_metric_id) {
+ if (cube->sampling_mode == mode && cube->max_n_cell == max_n_cell && cube->primary_metric_id == primary_metric_id) {
+ return FS_OK;
+ }
+ const struct metric_manifest *manifest = metric_manifest_manager_get_by_id(cube->manifest_manager, primary_metric_id);
+ if (manifest == NULL && mode != SAMPLING_MODE_COMPREHENSIVE) {
+ return FS_ERR_INVALID_METRIC_ID;
+ }
+ if ((mode == SAMPLING_MODE_TOPK && manifest->type != METRIC_TYPE_COUNTER) ||
+ (mode == SAMPLING_MODE_TOP_CARDINALITY && manifest->type != METRIC_TYPE_HLL)) {
+ return FS_ERR_INVALID_PARAM;
+ }
+
+ if (cube->primary_metric_id != -1) {
+ // delete previous settings
+ switch (cube->sampling_mode)
+ {
case SAMPLING_MODE_TOPK:
- cube->heavykeeper = heavy_keeper_new(max_n_cell);
- heavy_keeper_set_exdata_schema(cube->heavykeeper, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i);
+ heavy_keeper_free(cube->heavykeeper);
break;
case SAMPLING_MODE_COMPREHENSIVE:
- cube->table = hash_table_new(max_n_cell);
- hash_table_set_exdata_schema(cube->table, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i);
+ hash_table_free(cube->table);
+ break;
+ case SAMPLING_MODE_TOP_CARDINALITY:
+ spread_sketch_free(cube->spread_sketch);
break;
- case SAMPLING_MODE_TOP_CARDINALITY: {
- int width, depth;
- unsigned char precision;
- spread_sketch_get_parameter_recommendation(max_n_cell, &depth, &width, &precision);
- cube->spread_sketch = spread_sketch_new(depth, width, precision, 0, DUMMY_TIME_VAL);
- spread_sketch_set_exdata_schema(cube->spread_sketch, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i);
- break; }
default:
assert(0);
break;
+ }
}
- return cube;
+ switch (mode)
+ {
+ case SAMPLING_MODE_TOPK:
+ cube->heavykeeper = heavy_keeper_new(max_n_cell);
+ heavy_keeper_set_exdata_schema(cube->heavykeeper, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i);
+ break;
+ case SAMPLING_MODE_COMPREHENSIVE:
+ cube->table = hash_table_new(max_n_cell);
+ hash_table_set_exdata_schema(cube->table, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i);
+ break;
+ case SAMPLING_MODE_TOP_CARDINALITY: {
+ int width, depth;
+ unsigned char precision;
+ spread_sketch_get_parameter_recommendation(max_n_cell, &depth, &width, &precision);
+ cube->spread_sketch = spread_sketch_new(depth, width, precision, 0, DUMMY_TIME_VAL);
+ spread_sketch_set_exdata_schema(cube->spread_sketch, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i);
+ break; }
+ default:
+ assert(0);
+ break;
+ }
+
+ cube->sampling_mode = mode;
+ cube->max_n_cell = max_n_cell;
+ cube->primary_metric_id = primary_metric_id;
+
+ return FS_OK;
}
void cube_free(struct cube *cube) {
@@ -932,6 +970,7 @@ int cube_hll_add_field(struct cube *cube, int metric_id, const struct field *dim
}
int cube_counter_incrby(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, long long increment) {
+ assert(cube->primary_metric_id != -1);
assert(cube->sampling_mode == SAMPLING_MODE_COMPREHENSIVE ||
(cube->sampling_mode == SAMPLING_MODE_TOPK && (cube->primary_metric_id != metric_id || increment >= 0)) ||
(cube->sampling_mode == SAMPLING_MODE_TOP_CARDINALITY && cube->primary_metric_id != metric_id)
@@ -1020,8 +1059,10 @@ int cube_counter_set(struct cube *cube, int metric_id, const struct field *dimen
struct cube *cube_copy(const struct cube *cube)
{
- struct cube *cube_dup = cube_info_new(cube->cube_dimensions, cube->n_dimensions, cube->sampling_mode, cube->max_n_cell);
+ struct cube *cube_dup = cube_info_new(cube->cube_dimensions, cube->n_dimensions);
cube_dup->primary_metric_id = cube->primary_metric_id;
+ cube_dup->sampling_mode = cube->sampling_mode;
+ cube_dup->max_n_cell = cube->max_n_cell;
switch (cube->sampling_mode)
{
@@ -1087,8 +1128,10 @@ int cube_merge(struct cube *dest, const struct cube *src)
}
struct cube *cube_fork(const struct cube *cube) {
- struct cube *ret = cube_info_new(cube->cube_dimensions, cube->n_dimensions, cube->sampling_mode, cube->max_n_cell);
+ struct cube *ret = cube_info_new(cube->cube_dimensions, cube->n_dimensions);
ret->primary_metric_id = cube->primary_metric_id;
+ ret->sampling_mode = cube->sampling_mode;
+ ret->max_n_cell = cube->max_n_cell;
ret->manifest_manager = metric_manifest_manager_copy(cube->manifest_manager);
switch (cube->sampling_mode) {
@@ -1431,7 +1474,6 @@ struct field_list *cube_get_identifier(const struct cube *cube) {
const char *cube_get_metric_name(const struct cube *cube, int metric_id) {
const struct metric_manifest *metric = metric_manifest_manager_get_by_id(cube->manifest_manager, metric_id);
if (metric == NULL) {
- printf("metric is null\n");
return NULL;
}
diff --git a/src/cube.h b/src/cube.h
index 134ede8..842f064 100644
--- a/src/cube.h
+++ b/src/cube.h
@@ -13,7 +13,8 @@ extern "C"
struct cube;
struct cube_manager;
-struct cube *cube_new(const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, size_t max_n_cell);
+struct cube *cube_new(const struct field *dimensions, size_t n_dimensions);
+int cube_set_sampling_mode(struct cube *cube, enum sampling_mode mode, int max_n_cell, int primary_metric_id);
void cube_free(struct cube *cube);
void cube_reset(struct cube *cube);
struct cube *cube_copy(const struct cube *cube);
diff --git a/src/fieldstat.c b/src/fieldstat.c
index 4f4582b..002d754 100644
--- a/src/fieldstat.c
+++ b/src/fieldstat.c
@@ -48,7 +48,7 @@ void fieldstat_reset(struct fieldstat *instance)
cube_manager_reset(instance->cube_manager);
}
-int fieldstat_destroy_cube(struct fieldstat *instance, int cube_id)
+int fieldstat_cube_destroy(struct fieldstat *instance, int cube_id)
{
struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id);
if (cube == NULL) {
@@ -76,7 +76,24 @@ void fieldstat_free_tag_array(struct field *fields, size_t n_tags)
free(fields);
}
-int fieldstat_create_cube(struct fieldstat *instance, const struct field *cube_dimensions, size_t n_dimension, enum sampling_mode mode, size_t max_n_cell)
+// cppcheck-suppress [constParameterPointer, unmatchedSuppression]
+int fieldstat_cube_set_sampling(struct fieldstat *instance, int cube_id, enum sampling_mode mode, int max_n_cell, int primary_metric_id) {
+ if (max_n_cell <= 0) {
+ if (mode != SAMPLING_MODE_COMPREHENSIVE) {
+ return FS_ERR_INVALID_PARAM;
+ }
+ max_n_cell = INT32_MAX;
+ }
+
+ struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id);
+ if (cube == NULL) {
+ return FS_ERR_INVALID_CUBE_ID;
+ }
+
+ return cube_set_sampling_mode(cube, mode, max_n_cell, primary_metric_id);
+}
+
+int fieldstat_cube_create(struct fieldstat *instance, const struct field *cube_dimensions, size_t n_dimension)
{
if (instance == NULL) {
return FS_ERR_NULL_HANDLER;
@@ -86,14 +103,8 @@ int fieldstat_create_cube(struct fieldstat *instance, const struct field *cube_d
cube_dimensions = NULL;
n_dimension = 0;
}
- if (mode == SAMPLING_MODE_TOPK && max_n_cell == 0) {
- return FS_ERR_INVALID_PARAM;
- }
- if (max_n_cell == 0) {
- max_n_cell = INT32_MAX;
- }
-
- struct cube *cube = cube_new(cube_dimensions, n_dimension, mode, max_n_cell);
+
+ struct cube *cube = cube_new(cube_dimensions, n_dimension);
int ret = cube_manager_add(instance->cube_manager, cube);
if (ret < 0) {
cube_free(cube);
@@ -103,21 +114,6 @@ int fieldstat_create_cube(struct fieldstat *instance, const struct field *cube_d
return ret; //ret is the cube_id
}
-// cppcheck-suppress [constParameterPointer, unmatchedSuppression]
-int fieldstat_cube_set_primary_metric(struct fieldstat *instance, int cube_id, int metric_id)
-{
- if (instance == NULL) {
- return FS_ERR_NULL_HANDLER;
- }
-
- struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id);
- if (cube == NULL) {
- return FS_ERR_INVALID_CUBE_ID;
- }
-
- return cube_set_primary_metric(cube, metric_id);
-}
-
/* -------------------------------------------------------------------------- */
/* metric register */
/* -------------------------------------------------------------------------- */
diff --git a/src/fieldstat_easy.c b/src/fieldstat_easy.c
index 00a569a..46133f4 100644
--- a/src/fieldstat_easy.c
+++ b/src/fieldstat_easy.c
@@ -122,7 +122,8 @@ struct fieldstat_easy *fieldstat_easy_new(int max_thread_num, const char *name,
fse->fsu = malloc(sizeof(struct fs_easy_thread) * max_thread_num);
fse->max_thread_num = max_thread_num;
fse->delta = fieldstat_new();
- fieldstat_create_cube(fse->delta, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0);
+ fieldstat_cube_create(fse->delta, NULL, 0);
+ fieldstat_cube_set_sampling(fse->delta, 0, SAMPLING_MODE_COMPREHENSIVE, 0, 0);
fse->accumulate = fieldstat_fork(fse->delta);
fse->exporter = fieldstat_json_exporter_new();
diff --git a/test/profiling/main.c b/test/profiling/main.c
index 3af2b1a..8be10d3 100644
--- a/test/profiling/main.c
+++ b/test/profiling/main.c
@@ -31,7 +31,7 @@ int main () {
};
struct fieldstat *instance = fieldstat_new();
- fieldstat_create_cube(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0);
+ fieldstat_cube_create(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0);
fieldstat_register_counter(instance, "counter");
start = clock();
diff --git a/test/test_empty_tags.cpp b/test/test_empty_tags.cpp
index 967f8c0..8360eab 100644
--- a/test/test_empty_tags.cpp
+++ b/test/test_empty_tags.cpp
@@ -20,13 +20,13 @@ void assert_cell_null(const struct fieldstat *instance, int cube_id, long long e
fieldstat_tag_list_arr_free(tag_list, n_cell);
}
-
TEST(test_empty_tag, add_many_times)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 1);
-
+ int cube_id = fieldstat_cube_create(instance, NULL, 0);
int metric_id = fieldstat_register_counter(instance, cube_id, "metric");
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_COMPREHENSIVE, 1, 0);
+
fieldstat_counter_incrby(instance, cube_id, metric_id, NULL, 0, 1);
fieldstat_counter_incrby(instance, cube_id, metric_id, NULL, 0, 1);
@@ -38,8 +38,10 @@ TEST(test_empty_tag, add_many_times)
struct fieldstat *test_empty_my_init(enum sampling_mode mode = SAMPLING_MODE_COMPREHENSIVE)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, NULL, 0, mode, 1);
+ int cube_id = fieldstat_cube_create(instance, NULL, 0);
int metric_id = fieldstat_register_counter(instance, cube_id, "metric");
+ fieldstat_cube_set_sampling(instance, cube_id, mode, 1, 0);
+
fieldstat_counter_incrby(instance, cube_id, metric_id, NULL, 0, 1);
return instance;
@@ -97,8 +99,9 @@ TEST(test_empty_tag, merge_topk)
TEST(test_empty_tag, merge_spreadsketch)
{
struct fieldstat *instance_src = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance_src, NULL, 0, SAMPLING_MODE_TOP_CARDINALITY, 1);
+ int cube_id = fieldstat_cube_create(instance_src, NULL, 0);
int metric_id = fieldstat_register_hll(instance_src, cube_id, "metric", 4);
+ fieldstat_cube_set_sampling(instance_src, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 1, 0);
fieldstat_hll_add(instance_src, cube_id, metric_id, NULL, 0, "1", 1);
struct fieldstat *instance_dst = fieldstat_new();
diff --git a/test/test_exporter_json.cpp b/test/test_exporter_json.cpp
index 45a05cf..ebf6750 100644
--- a/test/test_exporter_json.cpp
+++ b/test/test_exporter_json.cpp
@@ -167,9 +167,10 @@ void topk_standard_oper(const std::function<void(Fieldstat_tag_list_wrapper *, u
void topk_init(struct fieldstat *instance, unsigned int test_expected_big_count)
{
const char *metric_name[TEST_METRIC_NUM] = {"topk1", "topk2"};
- int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_TOPK, TEST_TOPK_STANDARD_K);
+ int cube_id = fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3);
int m1 = fieldstat_register_counter(instance, cube_id, metric_name[0]);
int m2 = fieldstat_register_counter(instance, cube_id, metric_name[1]);
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, TEST_TOPK_STANDARD_K, 0);
std::function<void(Fieldstat_tag_list_wrapper *, unsigned int *)> topk_add =
[instance, cube_id, m1, m2](const Fieldstat_tag_list_wrapper *my_tags, unsigned int counts[TEST_METRIC_NUM]) {
@@ -186,12 +187,14 @@ TEST(export_test, cjson_export_with_fixed_tag_and_many_metrics_on_one_cube_of_co
// new instance
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, tag_list_num);
+ int cube_id = fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3);
int id_counter = fieldstat_register_counter(instance, cube_id, "counter");
int id_gauge = fieldstat_register_hll(instance, cube_id, "gauge", g_hll_standard->cfg.precision);
int id_histogram = fieldstat_register_histogram(instance, cube_id, "histogram",
g_histogram_standard->lowest_discernible_value, g_histogram_standard->highest_trackable_value, g_histogram_standard->significant_figures);
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_COMPREHENSIVE, tag_list_num, 0);
+
Fieldstat_tag_list_wrapper *fields[tag_list_num];
fill_random_tag(fields, tag_list_num);
@@ -289,9 +292,11 @@ extern "C" {
TEST(export_test, cjson_export_on_one_cube_of_spreadsketch_sampling) {
int K = 10;
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, K);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id = fieldstat_register_hll(instance, cube_id, "metric", 7);
int metric_count = fieldstat_register_counter(instance, cube_id, "oper cnt");
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, K, 0);
+
SpreadSketchZipfGenerator flow_generator(1.0, K * 10);
for (int i = 0; i < 100000; i++) {
@@ -342,11 +347,19 @@ TEST(export_test, empty_fieldstat_export_null) {
fieldstat_free(instance);
}
+int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k, int primary_metric_id=0)
+{
+ assert(mode == SAMPLING_MODE_COMPREHENSIVE);
+ int ret = fieldstat_cube_create(instance, dimensions, n_dimensions);
+ fieldstat_cube_set_sampling(instance, ret, mode, k, primary_metric_id);
+ return ret;
+}
+
TEST(export_test, only_registered_but_not_added_export_null_with_global_tag)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3);
+ int cube_id = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3);
fieldstat_register_counter(instance, cube_id, "counter");
fieldstat_register_hll(instance, cube_id, "gauge", g_hll_standard->cfg.precision);
fieldstat_register_histogram(instance, cube_id, "histogram",
@@ -362,9 +375,9 @@ TEST(export_test, only_registered_but_not_added_export_null_with_global_tag)
TEST(export_test, skip_two_empty_cube_and_export_last_one_with_global_tag)
{
struct fieldstat *instance = fieldstat_new();
- (void)fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3);
- (void)fieldstat_create_cube(instance, TEST_TAG_SHARED2, 3, SAMPLING_MODE_COMPREHENSIVE, 3);
- int cube_id_3 = fieldstat_create_cube(instance, TEST_TAG_SHARED3, 1, SAMPLING_MODE_COMPREHENSIVE, 3);
+ (void)test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3);
+ (void)test_fieldstat_cube_create(instance, TEST_TAG_SHARED2, 3, SAMPLING_MODE_COMPREHENSIVE, 3);
+ int cube_id_3 = test_fieldstat_cube_create(instance, TEST_TAG_SHARED3, 1, SAMPLING_MODE_COMPREHENSIVE, 3);
(void)fieldstat_register_counter(instance, cube_id_3, "counter");
(void)fieldstat_register_hll(instance, cube_id_3, "gauge", g_hll_standard->cfg.precision);
int id_histogram = fieldstat_register_histogram(instance, cube_id_3, "histogram",
@@ -410,9 +423,9 @@ TEST(export_test, skip_two_empty_cube_and_export_last_one_with_global_tag)
TEST(export_test, skip_empty_metrics_given_cube_deleted) {
struct fieldstat *instance = fieldstat_new();
- int cube_id_del = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3);
- int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED2, 3, SAMPLING_MODE_COMPREHENSIVE, 3);
- fieldstat_destroy_cube(instance, cube_id_del);
+ int cube_id_del = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3);
+ int cube_id = test_fieldstat_cube_create(instance, TEST_TAG_SHARED2, 3, SAMPLING_MODE_COMPREHENSIVE, 3);
+ fieldstat_cube_destroy(instance, cube_id_del);
(void)fieldstat_register_counter(instance, cube_id, "counter");
(void)fieldstat_register_counter(instance, cube_id, "counter2");
int metric_id = fieldstat_register_counter(instance, cube_id, "counter3");
@@ -459,7 +472,7 @@ TEST(export_test, skip_empty_metrics_given_cube_deleted) {
TEST(export_test, enable_delta_and_export_twice_without_new_metric)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10);
int id_counter = fieldstat_register_counter(instance, cube_id, "counter");
fieldstat_counter_incrby(instance, cube_id, id_counter, &TEST_TAG_INT, 1, 1);
@@ -528,7 +541,7 @@ TEST(export_test, enable_delta_and_export_twice_without_new_metric)
TEST(export_test, enable_delta_and_export_twice_with_new_metric_and_omit_histogram)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10);
int id_counter = fieldstat_register_counter(instance, cube_id, "counter");
int id_histogram = fieldstat_register_histogram(instance, cube_id, "histogram",
g_histogram_standard->lowest_discernible_value, g_histogram_standard->highest_trackable_value, g_histogram_standard->significant_figures);
@@ -610,7 +623,7 @@ TEST(export_test, enable_delta_and_export_twice_with_new_metric_and_omit_histogr
TEST(export_test, enable_delta_and_export_three_times_skipping_cube_with_no_counter)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10);
int id_counter = fieldstat_register_counter(instance, cube_id, "counter");
int id_histogram = fieldstat_register_histogram(instance, cube_id, "histogram",
g_histogram_standard->lowest_discernible_value, g_histogram_standard->highest_trackable_value, g_histogram_standard->significant_figures);
@@ -719,12 +732,12 @@ void test_check_delta_for_one_json(const struct field_list *expect_cell_tag, con
TEST(export_test, enable_delta_and_export_instance_with_many_cells_with_global_tags)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id1 = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id1 = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10);
int id_counter1 = fieldstat_register_counter(instance, cube_id1, "counter");
fieldstat_counter_incrby(instance, cube_id1, id_counter1, &TEST_TAG_INT, 1, 11);
fieldstat_counter_incrby(instance, cube_id1, id_counter1, &TEST_TAG_STRING, 1, 12);
- int cube_id2 = fieldstat_create_cube(instance, TEST_TAG_SHARED3, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id2 = test_fieldstat_cube_create(instance, TEST_TAG_SHARED3, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
int id_counter2 = fieldstat_register_counter(instance, cube_id2, "counter");
fieldstat_counter_incrby(instance, cube_id2, id_counter2, &TEST_TAG_INT, 1, 21);
fieldstat_counter_incrby(instance, cube_id2, id_counter2, &TEST_TAG_DOUBLE, 1, 22);
@@ -805,7 +818,7 @@ TEST(export_test, enable_delta_and_export_instance_with_many_cells_with_global_t
void test_reset_one_round(std::function<void(struct fieldstat *, struct fieldstat_json_exporter *)> trigger_reset)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 1);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 1);
int id_counter = fieldstat_register_counter(instance, cube_id, "counter");
fieldstat_counter_incrby(instance, cube_id, id_counter, &TEST_TAG_INT, 1, 11);
@@ -869,8 +882,8 @@ TEST(export_test, enable_delta_and_reset_on_change_exporter_tag) {
TEST(export_test, enable_delta_and_reset_on_delete_cube) {
auto trigger = [](struct fieldstat *instance, struct fieldstat_json_exporter *fieldstat_json_exporter) {
- fieldstat_destroy_cube(instance, 0);
- fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3);
+ fieldstat_cube_destroy(instance, 0);
+ test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3);
fieldstat_register_counter(instance, 0, "counter");
};
@@ -880,7 +893,7 @@ TEST(export_test, enable_delta_and_reset_on_delete_cube) {
TEST(export_test, delta_with_two_instance_same_config)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0);
+ int cube_id = test_fieldstat_cube_create(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0);
int id_counter = fieldstat_register_counter(instance, cube_id, "counter");
fieldstat_counter_incrby(instance, 0, id_counter, &TEST_TAG_INT, 1, 123);
int id_hist = fieldstat_register_histogram(instance, cube_id, "histogram", 1, 1000, 3);
@@ -920,7 +933,7 @@ TEST(export_test, delta_with_two_instance_same_config)
TEST(export_test, delta_with_two_instance_one_empty)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0);
+ int cube_id = test_fieldstat_cube_create(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0);
int id_counter = fieldstat_register_counter(instance, cube_id, "counter");
fieldstat_counter_incrby(instance, 0, id_counter, &TEST_TAG_INT, 1, 123);
int id_hist = fieldstat_register_histogram(instance, cube_id, "histogram", 1, 1000, 3);
@@ -959,7 +972,7 @@ TEST(export_test, delta_with_two_instance_one_empty)
TEST(export_test, delta_with_two_instance_different_cell)
{
struct fieldstat *instance = fieldstat_new();
- fieldstat_create_cube(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0);
+ test_fieldstat_cube_create(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0);
int id_counter = fieldstat_register_counter(instance, 0, "counter");
fieldstat_counter_incrby(instance, 0, id_counter, &TEST_TAG_INT, 1, 123);
@@ -1022,7 +1035,7 @@ TEST(export_test, export_flat_null_with_only_global_tag) {
TEST(export_test, export_flat_many_tags_many_cell) {
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 3);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 3);
int metric_id = fieldstat_register_counter(instance, cube_id, "counter metric");
diff --git a/test/test_fuzz_test.cpp b/test/test_fuzz_test.cpp
index 9eed5ed..aa3487a 100644
--- a/test/test_fuzz_test.cpp
+++ b/test/test_fuzz_test.cpp
@@ -77,11 +77,12 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_
// init cube
for (int i = 0; i < CUBE_NUM; i++) {
shared_tags[i] = new Fieldstat_tag_list_wrapper("shared_tag", i);
- int cube_id = fieldstat_create_cube(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count(), SAMPLING_MODE_COMPREHENSIVE, CELL_MAX);
+ int cube_id = fieldstat_cube_create(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count());
EXPECT_EQ(cube_id, i);
fieldstat_register_counter(master, cube_id, metric_name[METRIC_ID_COUNTER]);
fieldstat_register_hll(master, cube_id, metric_name[METRIC_ID_HLL], 6);
+ fieldstat_cube_set_sampling(master, cube_id, SAMPLING_MODE_COMPREHENSIVE, CELL_MAX, 0);
}
// all the possible fields
@@ -120,10 +121,12 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_
Fieldstat_tag_list_wrapper *new_tag = new Fieldstat_tag_list_wrapper("shared_tag", next_shared_tag_value++);
delete shared_tags[cube_id_to_change];
shared_tags[cube_id_to_change] = new_tag;
- fieldstat_destroy_cube(master, cube_id_to_change);
- int cube_id_new = fieldstat_create_cube(master, new_tag->get_tag(), new_tag->get_tag_count(), SAMPLING_MODE_COMPREHENSIVE, CELL_MAX);
+ fieldstat_cube_destroy(master, cube_id_to_change);
+ int cube_id_new = fieldstat_cube_create(master, new_tag->get_tag(), new_tag->get_tag_count());
fieldstat_register_counter(master, cube_id_new, metric_name[METRIC_ID_COUNTER]);
fieldstat_register_hll(master, cube_id_new, metric_name[METRIC_ID_HLL], 6);
+ fieldstat_cube_set_sampling(master, cube_id_new, SAMPLING_MODE_COMPREHENSIVE, CELL_MAX, 0);
+
EXPECT_EQ(cube_id_new, cube_id_to_change); // should new the cube in the hole leaved by the destroyed cube
// calibrate
for (int j = 0; j < INSTANCE_NUM; j++) {
@@ -220,9 +223,10 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_
// init cube
for (int i = 0; i < CUBE_NUM; i++) {
shared_tags[i] = new Fieldstat_tag_list_wrapper("shared_tag", i);
- int cube_id = fieldstat_create_cube(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count(), SAMPLING_MODE_TOPK, CELL_MAX);
+ int cube_id = fieldstat_cube_create(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count());
EXPECT_EQ(cube_id, i);
fieldstat_register_counter(master, cube_id, "topk");
+ fieldstat_cube_set_sampling(master, cube_id, SAMPLING_MODE_TOPK, CELL_MAX, 0);
}
// all the possible fields
@@ -258,9 +262,11 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_
Fieldstat_tag_list_wrapper *new_tag = new Fieldstat_tag_list_wrapper("shared_tag", next_shared_tag_value++);
delete shared_tags[cube_id_to_change];
shared_tags[cube_id_to_change] = new_tag;
- fieldstat_destroy_cube(master, cube_id_to_change);
- int cube_id_new = fieldstat_create_cube(master, new_tag->get_tag(), new_tag->get_tag_count(), SAMPLING_MODE_TOPK, CELL_MAX);
+ fieldstat_cube_destroy(master, cube_id_to_change);
+ int cube_id_new = fieldstat_cube_create(master, new_tag->get_tag(), new_tag->get_tag_count());
fieldstat_register_counter(master, cube_id_new, "topk");
+ fieldstat_cube_set_sampling(master, cube_id_new, SAMPLING_MODE_TOPK, CELL_MAX, 0);
+
EXPECT_EQ(cube_id_new, cube_id_to_change); // should new the cube in the hole leaved by the destroyed cube
// calibrate
for (int j = 0; j < INSTANCE_NUM; j++) {
@@ -361,9 +367,10 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_
// init cube
for (int i = 0; i < CUBE_NUM; i++) {
shared_tags[i] = new Fieldstat_tag_list_wrapper("shared_tag", i);
- int cube_id = fieldstat_create_cube(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count(), SAMPLING_MODE_TOP_CARDINALITY, CELL_MAX);
+ int cube_id = fieldstat_cube_create(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count());
EXPECT_EQ(cube_id, i);
fieldstat_register_hll(master, cube_id, "hll", 6);
+ fieldstat_cube_set_sampling(master, cube_id, SAMPLING_MODE_TOP_CARDINALITY, CELL_MAX, 0);
}
//init instance
@@ -392,10 +399,12 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_
Fieldstat_tag_list_wrapper *new_tag = new Fieldstat_tag_list_wrapper("shared_tag", next_shared_tag_value++);
delete shared_tags[cube_id_to_change];
shared_tags[cube_id_to_change] = new_tag;
- fieldstat_destroy_cube(master, cube_id_to_change);
- int cube_id_new = fieldstat_create_cube(master, new_tag->get_tag(), new_tag->get_tag_count(), SAMPLING_MODE_TOP_CARDINALITY, CELL_MAX);
+ fieldstat_cube_destroy(master, cube_id_to_change);
+ int cube_id_new = fieldstat_cube_create(master, new_tag->get_tag(), new_tag->get_tag_count());
fieldstat_register_hll(master, cube_id_new, "hll", 6);
EXPECT_EQ(cube_id_new, cube_id_to_change); // should new the cube in the hole leaved by the destroyed cube
+ fieldstat_cube_set_sampling(master, cube_id_new, SAMPLING_MODE_TOP_CARDINALITY, CELL_MAX, 0);
+
// calibrate
for (int j = 0; j < INSTANCE_NUM; j++) {
fieldstat_calibrate(master, replica[j]);
@@ -499,9 +508,11 @@ TEST(Fuzz_test, add_and_reset_with_randomly_generated_flows_and_randomly_chosen_
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance,NULL,0,SAMPLING_MODE_TOPK, 1); // K = 1, just to increase the possibility of FS_ERR_TOO_MANY_CELLS
+ int cube_id = fieldstat_cube_create(instance,NULL,0);
int primary_metric_id = fieldstat_register_counter(instance, cube_id, "counter");
int counter2_id = fieldstat_register_counter(instance, cube_id, "counter2");
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 1, 0); // K = 1, just to increase the possibility of FS_ERR_TOO_MANY_CELLS
+
fieldstat_counter_incrby(instance, cube_id, primary_metric_id, tag_list_wrapper[0]->get_tag(), tag_list_wrapper[0]->get_tag_count(), 1);
fieldstat_counter_incrby(instance, cube_id, counter2_id, tag_list_wrapper[0]->get_tag(), tag_list_wrapper[0]->get_tag_count(), 1);
@@ -550,9 +561,10 @@ TEST(perf, simple_one_for_perf_topk)
// init cube
for (int i = 0; i < CUBE_NUM; i++) {
shared_tags[i] = new Fieldstat_tag_list_wrapper("shared_tag", i);
- int cube_id = fieldstat_create_cube(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count(), SAMPLING_MODE_TOPK, CELL_MAX);
+ int cube_id = fieldstat_cube_create(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count());
EXPECT_EQ(cube_id, i);
fieldstat_register_counter(master, cube_id, "topk");
+ fieldstat_cube_set_sampling(master, cube_id, SAMPLING_MODE_TOPK, CELL_MAX, 0);
}
// init metric
@@ -598,8 +610,9 @@ TEST(perf, simple_one_for_perf_spreadsketch)
const int TEST_ROUND = 500000;
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_TAG_STRING, 1, SAMPLING_MODE_TOP_CARDINALITY, CELL_MAX);
+ int cube_id = fieldstat_cube_create(instance, &TEST_TAG_STRING, 1);
fieldstat_register_hll(instance, cube_id, "hll", 6);
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, CELL_MAX, 0);
SpreadSketchZipfGenerator generator(1.0, CELL_MAX * 10);
Fieldstat_tag_list_wrapper *cell_dimension[TEST_ROUND];
diff --git a/test/test_merge.cpp b/test/test_merge.cpp
index 6e8f802..5024cff 100644
--- a/test/test_merge.cpp
+++ b/test/test_merge.cpp
@@ -46,15 +46,23 @@ double merge_test_fieldstat_hll_get(const struct fieldstat *instance, int cube_i
return ret;
}
+int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k, int primary_metric_id=0)
+{
+ assert(mode == SAMPLING_MODE_COMPREHENSIVE);
+ int ret = fieldstat_cube_create(instance, dimensions, n_dimensions);
+ fieldstat_cube_set_sampling(instance, ret, mode, k, primary_metric_id);
+ return ret;
+}
+
TEST(unit_test_merge, cube_shared_tag_mapping_with_new_cube)
{
struct fieldstat *instance = fieldstat_new();
- (void)fieldstat_create_cube(instance, &TEST_TAG_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
- int cube_id2 = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ (void)test_fieldstat_cube_create(instance, &TEST_TAG_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id2 = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
int metric_id = fieldstat_register_counter(instance,cube_id2,"metric in cube 2");
fieldstat_counter_incrby(instance, cube_id2, metric_id, &TEST_TAG_STRING, 1, 1);
struct fieldstat *instance_dest = fieldstat_new();
- int cube_id_dest = fieldstat_create_cube(instance_dest, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id_dest = test_fieldstat_cube_create(instance_dest, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
fieldstat_merge(instance_dest, instance);
@@ -87,7 +95,7 @@ TEST(unit_test_merge, empty_instance)
TEST(unit_test_merge, new_cube_and_metric_to_empty_comprehensive)
{
struct fieldstat *instance = fieldstat_new();
- fieldstat_create_cube(instance, &TEST_TAG_INT, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ test_fieldstat_cube_create(instance, &TEST_TAG_INT, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
fieldstat_register_counter(instance, 0, "metric_name");
struct fieldstat *instance_dest = fieldstat_new();
@@ -108,7 +116,7 @@ TEST(unit_test_merge, new_cube_and_metric_to_empty_comprehensive)
TEST(unit_test_merge, new_cell_on_existing_cube_and_metric_comprehensive)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
int metric_id = fieldstat_register_counter(instance, 0, "metric_name");
struct fieldstat *instance_dest = fieldstat_new();
fieldstat_merge(instance_dest, instance);
@@ -140,7 +148,7 @@ TEST(unit_test_merge, new_cell_on_existing_cube_and_metric_comprehensive)
TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_comprehensive)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
int metric_id = fieldstat_register_counter(instance, cube_id, "metric_name");
fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, 5);
struct fieldstat *instance_dest = fieldstat_new();
@@ -159,7 +167,7 @@ TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_comprehens
TEST(unit_test_merge, new_too_many_cells_on_one_metric_given_source_cube_reset_and_get_different_cube_comprehensive)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2); // limit is 2
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2); // limit is 2
int metric_id = fieldstat_register_counter(instance, cube_id, "metric name");
fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, 1);
struct fieldstat *instance_dest = fieldstat_new();
@@ -187,7 +195,7 @@ TEST(unit_test_merge, new_too_many_cells_on_one_metric_given_source_cube_reset_a
TEST(unit_test_merge, new_too_many_cells_on_multiple_metric_given_source_cube_reset_and_get_different_cube_comprehensive)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2);
int metric_id1 = fieldstat_register_counter(instance, cube_id, "metric name1");
int metric_id2 = fieldstat_register_counter(instance, cube_id, "metric name2");
fieldstat_counter_incrby(instance, cube_id, metric_id1, &TEST_TAG_STRING, 1, 1); // 1st cell on metric name1
@@ -222,8 +230,9 @@ TEST(unit_test_merge, new_too_many_cells_on_multiple_metric_given_source_cube_re
TEST(unit_test_merge, new_cube_and_metric_to_empty_topk)
{
struct fieldstat *instance = fieldstat_new();
- fieldstat_create_cube(instance, &TEST_TAG_INT, 1, SAMPLING_MODE_TOPK, 10);
+ fieldstat_cube_create(instance, &TEST_TAG_INT, 1);
fieldstat_register_counter(instance, 0, "metric_name");
+ fieldstat_cube_set_sampling(instance, 0, SAMPLING_MODE_TOPK, 10, 0);
struct fieldstat *instance_dest = fieldstat_new();
@@ -243,8 +252,9 @@ TEST(unit_test_merge, new_cube_and_metric_to_empty_topk)
TEST(unit_test_merge, new_cell_on_existing_cube_and_metric_topk)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id = fieldstat_register_counter(instance, cube_id, "metric_name");
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, 0);
struct fieldstat *instance_dest = fieldstat_new();
fieldstat_merge(instance_dest, instance);
@@ -275,8 +285,9 @@ TEST(unit_test_merge, new_cell_on_existing_cube_and_metric_topk)
TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_topk)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id = fieldstat_register_counter(instance, cube_id, "metric_name");
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, 0);
fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, 5);
struct fieldstat *instance_dest = fieldstat_new();
@@ -306,8 +317,9 @@ TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_topk)
TEST(unit_test_merge, new_too_many_cells_on_one_metric_given_source_cube_reset_and_get_different_cube_topk)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 2);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id = fieldstat_register_counter(instance, cube_id, "metric name");
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 2, 0);
fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, 1);
struct fieldstat *instance_dest = fieldstat_new();
fieldstat_merge(instance_dest, instance);
@@ -334,8 +346,9 @@ TEST(unit_test_merge, new_too_many_cells_on_one_metric_given_source_cube_reset_a
struct fieldstat *topk_test_push_flows(vector<Fieldstat_tag_list_wrapper *> &flows_in_test, int K, long long count = 1)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, K);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id = fieldstat_register_counter(instance, cube_id, "metric name");
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, K, 0);
for (size_t i = 0; i < flows_in_test.size(); i++) {
fieldstat_counter_incrby(instance, cube_id, metric_id, flows_in_test[i]->get_tag(), flows_in_test[i]->get_tag_count(), count);
}
@@ -449,9 +462,10 @@ TEST(unit_test_merge, merge_accuracy_test_gen_dest_full_some_inserted_and_some_m
TEST(unit_test_merge, primary_metric_has_no_value)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 2);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_primary = fieldstat_register_counter(instance, cube_id, "primary");
int metric_operated = fieldstat_register_counter(instance, cube_id, "operated");
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 2, metric_primary);
fieldstat_counter_incrby(instance, cube_id, metric_operated, &TEST_TAG_STRING, 1, 1);
struct fieldstat *instance_dest = fieldstat_new();
fieldstat_merge(instance_dest, instance);
@@ -474,17 +488,19 @@ TEST(unit_test_merge, primary_metric_has_no_value)
TEST(unit_test_merge, primary_metric_id_different)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 2);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_primary = fieldstat_register_counter(instance, cube_id, "primary");
int metric_2 = fieldstat_register_counter(instance, cube_id, "2");
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 2, metric_primary);
+
fieldstat_counter_incrby(instance, cube_id, metric_primary, &TEST_TAG_STRING, 1, 100);
fieldstat_counter_incrby(instance, cube_id, metric_2, &TEST_TAG_STRING, 1, 1);
struct fieldstat *instance_dst = fieldstat_new();
- int cube_id_dst = fieldstat_create_cube(instance_dst, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 2);
+ int cube_id_dst = fieldstat_cube_create(instance_dst, &TEST_SHARED_TAG, 1);
fieldstat_register_counter(instance_dst, cube_id_dst, "2");
int metric_primary_dst = fieldstat_register_counter(instance_dst, cube_id_dst, "primary");
- fieldstat_cube_set_primary_metric(instance_dst, cube_id_dst, metric_primary_dst);
+ fieldstat_cube_set_sampling(instance_dst, cube_id_dst, SAMPLING_MODE_TOPK, 2, metric_primary_dst);
EXPECT_EQ(fieldstat_merge(instance_dst, instance), FS_ERR_INVALID_PARAM);
@@ -494,8 +510,9 @@ TEST(unit_test_merge, primary_metric_id_different)
TEST(unit_test_merge, new_cube_and_metric_to_empty_spreadsketch) {
struct fieldstat *instance = fieldstat_new();
- fieldstat_create_cube(instance, &TEST_TAG_INT, 1, SAMPLING_MODE_TOP_CARDINALITY, 10);
+ fieldstat_cube_create(instance, &TEST_TAG_INT, 1);
fieldstat_register_hll(instance, 0, "metric", 6);
+ fieldstat_cube_set_sampling(instance, 0, SAMPLING_MODE_TOP_CARDINALITY, 10, 0);
struct fieldstat *instance_dest = fieldstat_new();
fieldstat_merge(instance_dest, instance);
@@ -513,8 +530,9 @@ TEST(unit_test_merge, new_cube_and_metric_to_empty_spreadsketch) {
TEST(unit_test_merge, new_cell_on_existing_cube_and_metric_spreadsketch) {
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, 10);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id = fieldstat_register_hll(instance, cube_id, "metric", 6);
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 10, 0);
struct fieldstat *instance_dest = fieldstat_new();
fieldstat_merge(instance_dest, instance);
@@ -545,8 +563,9 @@ TEST(unit_test_merge, new_cell_on_existing_cube_and_metric_spreadsketch) {
TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_spreadsketch) {
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, 10);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id = fieldstat_register_hll(instance, cube_id, "metric", 6);
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 10, 0);
fieldstat_hll_add(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, "1", 1);
struct fieldstat *instance_dest = fieldstat_new();
@@ -569,8 +588,9 @@ TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_spreadsket
TEST(unit_test_merge, new_too_many_cells_on_one_metric_given_source_cube_reset_and_get_different_cube_spreadsketch) {
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, 2);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id = fieldstat_register_hll(instance, cube_id, "metric", 6);
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 2, 0);
fieldstat_hll_add(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, "1", 1);
struct fieldstat *instance_dest = fieldstat_new();
fieldstat_merge(instance_dest, instance);
@@ -602,8 +622,9 @@ TEST(unit_test_merge, gen_dest_full_all_src_inserted_given_src_flows_larger_spre
int K = 100;
SpreadSketchZipfGenerator flow_generator(1.0, K); // exactly the number of cells, so there will be almost all(in case of hash collision happen) cells added successfully
struct fieldstat *instance_src = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance_src, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, K);
+ int cube_id = fieldstat_cube_create(instance_src, &TEST_SHARED_TAG, 1);
int metric_id = fieldstat_register_hll(instance_src, cube_id, "metric", 6);
+ fieldstat_cube_set_sampling(instance_src, cube_id, SAMPLING_MODE_TOP_CARDINALITY, K, 0);
struct fieldstat *instance_dest = fieldstat_fork(instance_src);
const char dest_key[] = "key of dest";
const char src_key[] = "key of src";
@@ -660,8 +681,9 @@ TEST(unit_test_merge, merge_accuracy_test_gen_dest_full_some_inserted_and_some_m
int K = 10;
SpreadSketchZipfGenerator flow_generator(1.0, K * 10);
struct fieldstat *instance_src = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance_src, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, K);
+ int cube_id = fieldstat_cube_create(instance_src, &TEST_SHARED_TAG, 1);
int metric_id = fieldstat_register_hll(instance_src, cube_id, "metric", 6);
+ fieldstat_cube_set_sampling(instance_src, cube_id, SAMPLING_MODE_TOP_CARDINALITY, K, 0);
struct fieldstat *instance_dest = fieldstat_fork(instance_src);
std::unordered_map<std::string, std::unordered_set<std::string>> flow_cnt;
@@ -699,7 +721,7 @@ TEST(unit_test_merge, merge_accuracy_test_gen_dest_full_some_inserted_and_some_m
expected_unique_cnt[kv.first] = kv.second.size();
}
double recall = test_cal_topk_accuracy(test_result, expected_unique_cnt);
- EXPECT_GT(recall, 0.7);
+ EXPECT_GE(recall, 0.7);
printf("gen_dest_full_all_src_inserted_given_src_flows_larger_spreadsketch recall is %lf\n", recall);
fieldstat_free(instance_src);
diff --git a/test/test_metric_counter.cpp b/test/test_metric_counter.cpp
index d8ce16c..4c30b78 100644
--- a/test/test_metric_counter.cpp
+++ b/test/test_metric_counter.cpp
@@ -7,15 +7,18 @@
using namespace std;
+
struct fieldstat *test_init_standard_instance()
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
EXPECT_EQ(cube_id, 0);
int metric_id = fieldstat_register_counter(instance, cube_id, "czz_test counter metric");
EXPECT_EQ(metric_id, 0);
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_COMPREHENSIVE, 10, 0);
+
return instance;
}
@@ -26,6 +29,14 @@ long long my_fieldstat_counter_get(const struct fieldstat *instance, int cube_id
return ret;
}
+int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k, int primary_metric_id=0)
+{
+ assert(mode == SAMPLING_MODE_COMPREHENSIVE);
+ int ret = fieldstat_cube_create(instance, dimensions, n_dimensions);
+ fieldstat_cube_set_sampling(instance, ret, mode, k, primary_metric_id);
+ return ret;
+}
+
void test_assert_standard_instance(const struct fieldstat *instance)
{
int *ret_cube_id_arr = NULL;
@@ -110,8 +121,9 @@ TEST(metric_test_counter, merge_counter_twice_with_reset)
TEST(metric_test_counter, topk_add_and_test_accuracy)
{
struct fieldstat *instance = fieldstat_new();
- fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_TOPK, 10);
+ fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1);
fieldstat_register_counter(instance, 0, "test");
+ fieldstat_cube_set_sampling(instance, 0, SAMPLING_MODE_TOPK, 10, 0);
int tag_list_num = 10000;
Fieldstat_tag_list_wrapper *fields[tag_list_num];
@@ -165,7 +177,7 @@ TEST(metric_test_counter, topk_add_and_test_accuracy)
TEST(metric_test_counter, add_with_wrong_cube_id_expecting_fail)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_TOPK, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
int ret = fieldstat_counter_incrby(instance, cube_id + 1, 0, &TEST_TAG_INT, 1, 1);
EXPECT_EQ(ret, FS_ERR_INVALID_CUBE_ID);
@@ -178,7 +190,7 @@ TEST(metric_test_counter, add_with_wrong_cube_id_expecting_fail)
TEST(metric_test_counter, add_with_wrong_metric_id_expecting_fail)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_TOPK, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
int metric_id = fieldstat_register_counter(instance, 0, "test");
int ret = fieldstat_counter_incrby(instance, cube_id, metric_id + 1, &TEST_TAG_INT, 1, 1);
@@ -192,9 +204,10 @@ TEST(metric_test_counter, add_with_wrong_metric_id_expecting_fail)
TEST(metric_test_counter, add_and_query_on_dummy_cell_of_topk)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
fieldstat_register_counter(instance, cube_id, "primary"); // also the dummy one
int metric_id = fieldstat_register_counter(instance, cube_id, "using");
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, 0);
fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1);
@@ -208,11 +221,10 @@ TEST(metric_test_counter, add_and_query_on_dummy_cell_of_topk)
TEST(metric_test_counter, primary_counter_add_after_first)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id_primary = fieldstat_register_counter(instance, cube_id, "primary");
int metric_id2 = fieldstat_register_counter(instance, cube_id, "using");
- fieldstat_cube_set_primary_metric(instance, cube_id, metric_id_primary);
-
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, metric_id_primary);
int ret = fieldstat_counter_incrby(instance, cube_id, metric_id2, &TEST_TAG_INT, 1, 10);
EXPECT_EQ(ret, FS_OK);
diff --git a/test/test_metric_histogram.cpp b/test/test_metric_histogram.cpp
index 3a28aa3..3ef171c 100644
--- a/test/test_metric_histogram.cpp
+++ b/test/test_metric_histogram.cpp
@@ -6,10 +6,18 @@
#include "hdr/hdr_histogram.h"
#include "histogram_encoder.h"
+int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k)
+{
+ assert(mode == SAMPLING_MODE_COMPREHENSIVE);
+ int ret = fieldstat_cube_create(instance, dimensions, n_dimensions);
+ fieldstat_cube_set_sampling(instance, ret, mode, k, 0);
+ return ret;
+}
+
struct fieldstat *test_init_standard_instance_one_cube_one_metric_one_cell_hdr()
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
EXPECT_EQ(cube_id, 0);
int metric_id = fieldstat_register_histogram(instance, cube_id, "czz_test hdr metric", 1, 600000, 3);
@@ -107,7 +115,7 @@ TEST(metric_test_histogram, merge_twice_with_reset)
TEST(metric_test_histogram, add_with_wrong_cube_id_expecting_fail)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
int ret = fieldstat_histogram_record(instance, cube_id + 1, 0, &TEST_TAG_INT, 1, 1);
EXPECT_EQ(ret, FS_ERR_INVALID_CUBE_ID);
@@ -120,7 +128,7 @@ TEST(metric_test_histogram, add_with_wrong_cube_id_expecting_fail)
TEST(metric_test_histogram, add_with_wrong_metric_id_expecting_fail)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
int metric_id = fieldstat_register_histogram(instance, cube_id, "czz_test", 1, 600000, 3);
int ret = fieldstat_histogram_record(instance, cube_id, metric_id + 1, &TEST_TAG_INT, 1, 1);
@@ -169,7 +177,7 @@ TEST(metric_test_histogram, encode_decode_b64)
TEST(metric_test_histogram, can_add_0value) // histogram only allow min_val > 0, but it can accept value == 0
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
int metric_id = fieldstat_register_histogram(instance, cube_id, "czz_test", 1, 600000, 3);
int ret = fieldstat_histogram_record(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 0);
diff --git a/test/test_metric_hll.cpp b/test/test_metric_hll.cpp
index 204c762..713c00d 100644
--- a/test/test_metric_hll.cpp
+++ b/test/test_metric_hll.cpp
@@ -8,10 +8,18 @@
#include "fieldstat.h"
#include "utils.hpp"
+int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k)
+{
+ assert(mode == SAMPLING_MODE_COMPREHENSIVE);
+ int ret = fieldstat_cube_create(instance, dimensions, n_dimensions);
+ fieldstat_cube_set_sampling(instance, ret, mode, k, 0);
+ return ret;
+}
+
struct fieldstat *test_init_standard_instance_one_cube_one_metric_one_cell_hll(bool is_gauge = false)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
EXPECT_EQ(cube_id, 0);
int metric_id = fieldstat_register_hll(instance, cube_id, "czz_test hll metric", 10);
@@ -198,7 +206,7 @@ TEST(metric_test_hll, serialize_with_b64_and_query_with_python_api)
TEST(metric_test_hll, add_with_wrong_cube_id_expecting_fail)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
int ret = fieldstat_hll_add(instance, cube_id + 1, 0, &TEST_TAG_INT, 1, "hello", 5);
EXPECT_EQ(ret, FS_ERR_INVALID_CUBE_ID);
@@ -211,7 +219,7 @@ TEST(metric_test_hll, add_with_wrong_cube_id_expecting_fail)
TEST(metric_test_hll, add_with_wrong_metric_id_expecting_fail)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
int metric_id = fieldstat_register_hll(instance, cube_id, "czz_test hll metric", 10);
int ret = fieldstat_hll_add(instance, cube_id, metric_id + 1, &TEST_TAG_INT, 1, "hello", 5);
@@ -226,8 +234,9 @@ TEST(metric_test_hll, spread_sketch_add_and_test_accuracy)
{
struct fieldstat *instance = fieldstat_new();
int K = 10;
- fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_TOP_CARDINALITY, K);
+ fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1);
fieldstat_register_hll(instance, 0, "testss", 6);
+ fieldstat_cube_set_sampling(instance, 0, SAMPLING_MODE_TOP_CARDINALITY, K, 0);
int n_flows = 100000;
std::unordered_map<std::string, std::unordered_set<std::string>> flow_cnt;
diff --git a/test/test_performance.cpp b/test/test_performance.cpp
index b1ada45..d0ca633 100644
--- a/test/test_performance.cpp
+++ b/test/test_performance.cpp
@@ -6,6 +6,15 @@
#include "fieldstat_exporter.h"
#include "utils.hpp"
+
+int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *tag, size_t tag_count, enum sampling_mode mode, int k, int primary_metric_id=0)
+{
+ int ret = fieldstat_cube_create(instance, tag, tag_count);
+ fieldstat_cube_set_sampling(instance, ret, mode, k, primary_metric_id);
+ return ret;
+}
+
+
// /* -------------------------------------------------------------------------- */
// /* merge */
// /* -------------------------------------------------------------------------- */
@@ -18,8 +27,9 @@ double perform_merge_test(std::function<void (struct fieldstat*, int, int, const
fields[i] = new Fieldstat_tag_list_wrapper("my key", i);
}
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, mode, MAX_CELL_NUM);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id = metric_register_func(instance);
+ fieldstat_cube_set_sampling(instance, cube_id, mode, MAX_CELL_NUM, metric_id);
for (int j = 0; j < MAX_CELL_NUM; j++) {
metric_add_func(instance, cube_id, metric_id, fields[j]->get_tag(), 1);
}
@@ -163,11 +173,13 @@ struct fieldstat *construct_fs_with_many_empty_cubes(int cube_num, int metric_nu
struct field tmp_tag = TEST_TAG_INT;
for (int i = 0; i < cube_num; i++) {
tmp_tag.value_longlong = i;
- int cube_id = fieldstat_create_cube(instance, &tmp_tag, 1, mode, 1000);
+ int cube_id = fieldstat_cube_create(instance, &tmp_tag, 1);
for (int j = 0; j < metric_num; j++) {
fieldstat_register_counter(instance, cube_id, std::to_string(j).c_str());
}
+
+ fieldstat_cube_set_sampling(instance, cube_id, mode, 1000, 0);
}
return instance;
}
@@ -219,7 +231,7 @@ TEST(test_performance, performance_test_add_cells_comprehensive)
}
// getchar();
struct fieldstat *instance = fieldstat_new();
- fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, cell_count);
+ test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, cell_count);
fieldstat_register_counter(instance, 0, "test");
clock_t start = clock();
@@ -246,8 +258,9 @@ TEST(test_performance, performance_test_add_cells_topk)
fields[i].value_longlong = rand() % 1000;
}
struct fieldstat *instance = fieldstat_new();
- fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_TOPK, 1000);
+ fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1);
fieldstat_register_counter(instance, 0, "test");
+ fieldstat_cube_set_sampling(instance, 0, SAMPLING_MODE_TOPK, 1000, 0);
// getchar();
clock_t start = clock();
@@ -267,7 +280,7 @@ TEST(test_performance, performance_test_add_cells_topk)
TEST(test_performance, performance_test_add_cells_histogram_record)
{
struct fieldstat *instance = fieldstat_new();
- fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
fieldstat_register_histogram(instance, 0, "test", 1, 100000, 3);
size_t test_num = 100000;
long long vals[test_num];
@@ -288,7 +301,7 @@ TEST(test_performance, performance_test_add_cells_histogram_record)
TEST(test_performance, performance_test_add_cells_hll_add)
{
struct fieldstat *instance = fieldstat_new();
- fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
fieldstat_register_hll(instance, 0, "test", 6);
size_t test_num = 100000;
std::string vals[test_num];
@@ -325,7 +338,7 @@ TEST(test_performance, performance_test_add_cells_comprehensive_5_tags)
}
// getchar();
struct fieldstat *instance = fieldstat_new();
- fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, cell_count);
+ test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, cell_count);
fieldstat_register_counter(instance, 0, "test");
clock_t start = clock();
@@ -346,7 +359,7 @@ TEST(test_performance, performance_test_add_cells_comprehensive_5_tags)
TEST(test_performance, performance_test_add_cells_histogram_record_5tags)
{
struct fieldstat *instance = fieldstat_new();
- fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
fieldstat_register_histogram(instance, 0, "test", 1, 100000, 3);
size_t test_num = 100000;
long long vals[test_num];
@@ -374,7 +387,7 @@ TEST(test_performance, performance_test_add_cells_histogram_record_5tags)
TEST(test_performance, performance_test_add_cells_hll_add_5tags)
{
struct fieldstat *instance = fieldstat_new();
- fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ test_fieldstat_cube_create(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
fieldstat_register_hll(instance, 0, "test", 6);
size_t test_num = 100000;
std::string vals[test_num];
@@ -419,7 +432,7 @@ TEST(test_performance, export_many_cells)
struct fieldstat *instance = fieldstat_new();
for (int i = 0; i < CUBE_NUM; i++) {
Fieldstat_tag_list_wrapper cube_tag("shared key", i);
- int cube_id = fieldstat_create_cube(instance, cube_tag.get_tag(), cube_tag.get_tag_count(), SAMPLING_MODE_COMPREHENSIVE, MAX_CELL_NUM);
+ int cube_id = test_fieldstat_cube_create(instance, cube_tag.get_tag(), cube_tag.get_tag_count(), SAMPLING_MODE_COMPREHENSIVE, MAX_CELL_NUM);
for (int j = 0; j < METRIC_NUM; j++) {
string metric_name = "metric name" + to_string(i) + to_string(j);
@@ -493,10 +506,10 @@ TEST(test_performance, callibrate_unchanged)
fieldstat_free(instance_dest);
}
-struct fieldstat *construct_fs_with_many_cells(int cell_num, enum sampling_mode mode)
+struct fieldstat *construct_fs_with_many_cells(int cell_num)
{
struct fieldstat *instance = fieldstat_new();
- fieldstat_create_cube(instance, &TEST_TAG_INT, 1, mode, cell_num);
+ test_fieldstat_cube_create(instance, &TEST_TAG_INT, 1, SAMPLING_MODE_COMPREHENSIVE, cell_num);
fieldstat_register_counter(instance, 0, "test");
struct field tmp_tag = TEST_TAG_INT;
for (int i = 0; i < cell_num; i++) {
@@ -509,7 +522,7 @@ struct fieldstat *construct_fs_with_many_cells(int cell_num, enum sampling_mode
TEST(test_performance, reset_many_cells)
{
- struct fieldstat *instance = construct_fs_with_many_cells(1000, SAMPLING_MODE_COMPREHENSIVE); // many empty cubes, 10 metrics is a common case
+ struct fieldstat *instance = construct_fs_with_many_cells(1000); // many empty cubes, 10 metrics is a common case
clock_t start = clock();
fieldstat_reset(instance);
clock_t end = clock();
diff --git a/test/test_register_and_reset.cpp b/test/test_register_and_reset.cpp
index 31f57ba..380b156 100644
--- a/test/test_register_and_reset.cpp
+++ b/test/test_register_and_reset.cpp
@@ -3,13 +3,20 @@
#include "fieldstat.h"
#include "utils.hpp"
+int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k)
+{
+ assert(mode == SAMPLING_MODE_COMPREHENSIVE);
+ int ret = fieldstat_cube_create(instance, dimensions, n_dimensions);
+ fieldstat_cube_set_sampling(instance, ret, mode, k, 0);
+ return ret;
+}
TEST(test_register, delete_cube_and_register_and_origin_position)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
- fieldstat_destroy_cube(instance, cube_id);
- cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ fieldstat_cube_destroy(instance, cube_id);
+ cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
EXPECT_EQ(cube_id, 0);
EXPECT_EQ(fieldstat_find_cube(instance, &TEST_SHARED_TAG, 1) , 0);
fieldstat_free(instance);
@@ -18,11 +25,11 @@ TEST(test_register, delete_cube_and_register_and_origin_position)
TEST(test_register, delete_comprehensive_cube_with_cells_and_metrics)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
int metric_id = fieldstat_register_counter(instance, cube_id, "counter");
fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1);
- fieldstat_destroy_cube(instance, cube_id);
+ fieldstat_cube_destroy(instance, cube_id);
struct field_list *tag_list = fieldstat_cube_get_tags(instance, cube_id);
EXPECT_EQ(tag_list, nullptr);
@@ -35,11 +42,12 @@ TEST(test_register, delete_comprehensive_cube_with_cells_and_metrics)
TEST(test_register, delete_topk_cube_with_cells_and_metrics)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id = fieldstat_register_counter(instance, cube_id, "counter");
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, metric_id);
fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1);
- fieldstat_destroy_cube(instance, cube_id);
+ fieldstat_cube_destroy(instance, cube_id);
struct field_list *tag_list = fieldstat_cube_get_tags(instance, cube_id);
EXPECT_EQ(tag_list, nullptr);
int cube_id_ret = fieldstat_find_cube(instance, &TEST_SHARED_TAG, 1);
@@ -51,14 +59,15 @@ TEST(test_register, delete_topk_cube_with_cells_and_metrics)
TEST(test_register, delete_spreadsketch_cube_with_cells_and_metrics)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, 10);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id1 = fieldstat_register_counter(instance, cube_id, "counter");
int metric_primary = fieldstat_register_hll(instance, cube_id, "hll_primary", 5);
- fieldstat_cube_set_primary_metric(instance, cube_id, metric_primary);
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 10, metric_primary);
+
fieldstat_counter_incrby(instance, cube_id, metric_id1, &TEST_TAG_INT, 1, 1);
fieldstat_hll_add_field(instance, cube_id, metric_primary, &TEST_TAG_INT, 1, &TEST_TAG_DOUBLE, 1);
- fieldstat_destroy_cube(instance, cube_id);
+ fieldstat_cube_destroy(instance, cube_id);
struct field_list *tag_list = fieldstat_cube_get_tags(instance, cube_id);
EXPECT_EQ(tag_list, nullptr);
int cube_id_ret = fieldstat_find_cube(instance, &TEST_SHARED_TAG, 1);
@@ -67,7 +76,6 @@ TEST(test_register, delete_spreadsketch_cube_with_cells_and_metrics)
fieldstat_free(instance);
}
-
int test_get_max_metric_id(const struct fieldstat *instance)
{
int *metric_id_out;
@@ -80,7 +88,7 @@ int test_get_max_metric_id(const struct fieldstat *instance)
TEST(test_register, reset_and_try_to_query_cell_comprehensive)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
int metric_id = fieldstat_register_counter(instance, cube_id, "counter");
fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1);
@@ -99,8 +107,10 @@ TEST(test_register, reset_and_try_to_query_cell_comprehensive)
TEST(test_register, reset_and_try_to_query_cell_topk)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id = fieldstat_register_counter(instance, cube_id, "counter");
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, metric_id);
+
fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1);
fieldstat_reset(instance);
@@ -118,8 +128,9 @@ TEST(test_register, reset_and_try_to_query_cell_topk)
TEST(test_register, reset_and_try_to_query_cell_spreadsketch)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, 10);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id = fieldstat_register_hll(instance, cube_id, "hll", 5);
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 10, metric_id);
fieldstat_hll_add(instance, cube_id, metric_id, &TEST_TAG_INT, 1, "12abc", 5);
fieldstat_reset(instance);
@@ -137,7 +148,7 @@ TEST(test_register, reset_and_try_to_query_cell_spreadsketch)
TEST(test_register, reset_and_new_cell_comprehensive)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2);
int metric_id = fieldstat_register_counter(instance, cube_id, "counter");
fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1);
fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_DOUBLE, 1, 1);
@@ -154,8 +165,9 @@ TEST(test_register, reset_and_new_cell_comprehensive)
TEST(test_register, reset_and_new_cell_topk)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 1);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id = fieldstat_register_counter(instance, cube_id, "counter");
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 1, metric_id);
fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 100);//100: bigger value
int ret = fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_DOUBLE, 1, 1);
EXPECT_EQ(ret, FS_ERR_TOO_MANY_CELLS);
@@ -170,8 +182,9 @@ TEST(test_register, reset_and_new_cell_topk)
TEST(test_register, reset_and_new_cell_spreadsketch)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, 1);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id = fieldstat_register_hll(instance, cube_id, "hll", 5);
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 1, metric_id);
// spread sketch will store more data than expected cell number 1. So loop for many cells first to trigger the error
struct field test_tag_long = TEST_TAG_INT;
for (int i = 0; i < 10000; i++) {
@@ -191,7 +204,7 @@ TEST(test_register, reset_and_new_cell_spreadsketch)
TEST(test_register, ensure_recovery_more_faster_comprehensive) {
struct fieldstat *instance = fieldstat_new();
int cell_num = 1000;
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, cell_num);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, cell_num);
int metric_id = fieldstat_register_counter(instance, cube_id, "counter");
struct field test_tag_long = TEST_TAG_INT;
@@ -221,8 +234,10 @@ TEST(test_register, ensure_recovery_more_faster_comprehensive) {
TEST(test_register, ensure_recovery_more_faster_topk) {
struct fieldstat *instance = fieldstat_new();
int cell_num = 1000;
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, cell_num);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id = fieldstat_register_counter(instance, cube_id, "counter");
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, cell_num, metric_id);
+
struct field test_tag_long = TEST_TAG_INT;
clock_t start = clock();
@@ -251,8 +266,9 @@ TEST(test_register, ensure_recovery_more_faster_topk) {
TEST(test_register, ensure_recovery_more_faster_spreadsketch) {
struct fieldstat *instance = fieldstat_new();
int cell_num = 1000;
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOP_CARDINALITY, cell_num);
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
int metric_id = fieldstat_register_hll(instance, cube_id, "counter", 6);
+ fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, cell_num, metric_id);
struct field test_tag_long = TEST_TAG_INT;
clock_t start = clock();
@@ -285,7 +301,7 @@ TEST(test_register, register_many_cubes)
struct field shared_tag = TEST_SHARED_TAG;
for (int i = 0; i < registered_cube; i++) {
shared_tag.value_longlong = i;
- int cube_id = fieldstat_create_cube(instance, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
EXPECT_EQ(cube_id, i);
fieldstat_register_counter(instance, cube_id, "counter");
}
@@ -306,7 +322,7 @@ TEST(test_register, register_many_cubes)
TEST(test_register, add_many_tagged_cells)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
size_t n_field = 1000;
struct field test_tag_long[n_field];
for (size_t i = 0; i < n_field; i++) {
@@ -327,7 +343,7 @@ TEST(test_register, add_many_tagged_cells)
TEST(test_register, add_long_tagged_cells)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
struct field test_tag_long = TEST_TAG_STRING;
char *long_string = (char *)malloc(5001);
memset(long_string, 'a', 5001);
@@ -349,7 +365,7 @@ TEST(test_register, add_long_tagged_cells)
TEST(test_register, register_many_metrics)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
int metric_id = 0;
for (int i = 0; i < 200; i++) {
metric_id = fieldstat_register_counter(instance, cube_id, (std::string("counter ") + std::to_string(i)).c_str());
@@ -381,13 +397,13 @@ TEST(test_register, fork_registered_info_from_empty_one)
TEST(test_register, fork_registered_info_with_cube_and_metric)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
int metric_id = fieldstat_register_counter(instance, cube_id, "counter");
int metric_id2 = fieldstat_register_counter(instance, cube_id, "counter2");
fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1);
- int cube_id_del = fieldstat_create_cube(instance, &TEST_FIELD_VALUE_DOUBLE_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
- fieldstat_create_cube(instance, &TEST_TAG_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
- fieldstat_destroy_cube(instance, cube_id_del);
+ int cube_id_del = test_fieldstat_cube_create(instance, &TEST_FIELD_VALUE_DOUBLE_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ test_fieldstat_cube_create(instance, &TEST_TAG_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ fieldstat_cube_destroy(instance, cube_id_del);
struct fieldstat *dup = fieldstat_fork(instance);
@@ -423,8 +439,8 @@ TEST(test_register, unregister_cube_on_wrong_instance)
struct fieldstat *instance = fieldstat_new();
struct fieldstat *instance_dst = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
- int cube_id2 = fieldstat_create_cube(instance, &TEST_TAG_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id2 = test_fieldstat_cube_create(instance, &TEST_TAG_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
int metric_id = fieldstat_register_counter(instance, cube_id, "counter");
int metric_id2 = fieldstat_register_counter(instance, cube_id2, "counter2");
fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1);
@@ -432,7 +448,7 @@ TEST(test_register, unregister_cube_on_wrong_instance)
fieldstat_merge(instance_dst, instance);
- fieldstat_destroy_cube(instance_dst, 0);
+ fieldstat_cube_destroy(instance_dst, 0);
int *cube_ids;
int cube_num;
@@ -457,7 +473,7 @@ TEST(test_register, unregister_cube_on_wrong_instance)
TEST(test_register, register_many_cells_on_unlimited_sized_cube)
{
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 0);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 0);
int metric_id = fieldstat_register_counter(instance, cube_id, "counter");
struct field test_tag = {"abc", FIELD_VALUE_INTEGER, {.value_longlong = 0}};
for (int i = 0; i < 10000; i++) {
@@ -478,15 +494,15 @@ TEST(test_register, register_many_cells_on_unlimited_sized_cube)
TEST(test_register, register_cube_twice) {
struct fieldstat *instance = fieldstat_new();
- fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10);
- int cube_id2 = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10);
+ test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id2 = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
EXPECT_EQ(cube_id2, FS_ERR_INVALID_KEY);
fieldstat_free(instance);
}
TEST(test_register, find_cube) {
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
int find_cube_id = fieldstat_find_cube(instance, &TEST_SHARED_TAG, 1);
EXPECT_EQ(cube_id, find_cube_id);
@@ -498,7 +514,7 @@ TEST(test_register, find_cube) {
TEST(test_register, register_metric_twice) {
struct fieldstat *instance = fieldstat_new();
- int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
fieldstat_register_counter(instance, cube_id, "counter");
int metric_id2 = fieldstat_register_counter(instance, cube_id, "counter");
@@ -506,10 +522,20 @@ TEST(test_register, register_metric_twice) {
fieldstat_free(instance);
}
+TEST(test_register, add_on_uninitialized_cube) {
+ struct fieldstat *instance = fieldstat_new();
+ int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1);
+ int metric_id = fieldstat_register_counter(instance, cube_id, "counter");
+ // fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1);
+ EXPECT_DEATH(fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1), ".*");
+
+ fieldstat_free(instance);
+}
+
TEST(calibrate, target_one_more_metric)
{
struct fieldstat *master = fieldstat_new();
- int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
fieldstat_register_counter(master, cube_id, "counter");
struct fieldstat *target = fieldstat_fork(master);
EXPECT_EQ(fieldstat_register_counter(target, cube_id, "counter2"), 1);
@@ -529,7 +555,7 @@ TEST(calibrate, target_one_more_metric)
TEST(calibrate, master_one_more_metric)
{
struct fieldstat *master = fieldstat_new();
- int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
fieldstat_register_counter(master, cube_id, "counter");
struct fieldstat *target = fieldstat_fork(master);
EXPECT_EQ(fieldstat_register_counter(master, cube_id, "counter2"), 1);
@@ -551,7 +577,7 @@ TEST(calibrate, master_one_more_metric)
TEST(calibrate, different_metric)
{
struct fieldstat *master = fieldstat_new();
- int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
fieldstat_register_counter(master, cube_id, "counter");
struct fieldstat *target = fieldstat_fork(master);
EXPECT_EQ(fieldstat_register_counter(target, cube_id, "counter2"), 1);
@@ -573,10 +599,10 @@ TEST(calibrate, different_metric)
TEST(calibrate, target_more_cube)
{
struct fieldstat *master = fieldstat_new();
- int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
fieldstat_register_counter(master, cube_id, "counter");
struct fieldstat *target = fieldstat_fork(master);
- int cube_id2 = fieldstat_create_cube(target, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id2 = test_fieldstat_cube_create(target, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
EXPECT_EQ(cube_id2, 1);
fieldstat_register_counter(target, cube_id, "counter2");
@@ -605,9 +631,9 @@ TEST(calibrate, target_more_cube)
TEST(calibrate, master_more_cube)
{
struct fieldstat *master = fieldstat_new();
- int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
struct fieldstat *target = fieldstat_fork(master);
- int cube_id2 = fieldstat_create_cube(master, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id2 = test_fieldstat_cube_create(master, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
fieldstat_calibrate(master, target);
@@ -636,10 +662,10 @@ TEST(calibrate, master_more_cube)
TEST(calibrate, master_change_cube)
{
struct fieldstat *master = fieldstat_new();
- int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
struct fieldstat *target = fieldstat_fork(master);
- fieldstat_destroy_cube(master, cube_id);
- fieldstat_create_cube(master, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ fieldstat_cube_destroy(master, cube_id);
+ test_fieldstat_cube_create(master, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
fieldstat_calibrate(master, target);
@@ -660,14 +686,14 @@ TEST(calibrate, master_many_cube)
struct field shared_tag = TEST_SHARED_TAG;
for (int i = 0; i < registered_cube; i++) {
shared_tag.value_longlong = i;
- int cube_id = fieldstat_create_cube(master, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(master, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
EXPECT_EQ(cube_id, i);
}
struct fieldstat *target = fieldstat_fork(master);
int new_registered = 5000;
for (int i = 0; i < new_registered; i++) {
shared_tag.value_longlong = i + registered_cube;
- int cube_id = fieldstat_create_cube(master, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
+ int cube_id = test_fieldstat_cube_create(master, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10);
EXPECT_EQ(cube_id, i + registered_cube);
}
@@ -707,13 +733,13 @@ TEST(calibrate, issue_calibrate_wrong)
struct fieldstat *master = fieldstat_new();
const struct field *tag_A = &TEST_SHARED_TAG;
const struct field *tag_B = &TEST_TAG_INT;
- int cube_idA = fieldstat_create_cube(master, tag_A, 1, SAMPLING_MODE_COMPREHENSIVE, 1);
- int cube_idB = fieldstat_create_cube(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1);
+ int cube_idA = test_fieldstat_cube_create(master, tag_A, 1, SAMPLING_MODE_COMPREHENSIVE, 1);
+ int cube_idB = test_fieldstat_cube_create(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1);
struct fieldstat *target = fieldstat_fork(master);
- EXPECT_EQ(fieldstat_destroy_cube(master, cube_idA), FS_OK);
- EXPECT_EQ(fieldstat_destroy_cube(master, cube_idB), FS_OK);
- int cube_idBa = fieldstat_create_cube(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1);
+ EXPECT_EQ(fieldstat_cube_destroy(master, cube_idA), FS_OK);
+ EXPECT_EQ(fieldstat_cube_destroy(master, cube_idB), FS_OK);
+ int cube_idBa = test_fieldstat_cube_create(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1);
EXPECT_EQ(cube_idBa, 0);
fieldstat_calibrate(master, target);
@@ -739,11 +765,11 @@ TEST(calibrate, delete_first_cube)
struct fieldstat *master = fieldstat_new();
const struct field *tag_A = &TEST_SHARED_TAG;
const struct field *tag_B = &TEST_TAG_INT;
- int cube_idA = fieldstat_create_cube(master, tag_A, 1, SAMPLING_MODE_COMPREHENSIVE, 1);
- int cube_idB = fieldstat_create_cube(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1);
+ int cube_idA = test_fieldstat_cube_create(master, tag_A, 1, SAMPLING_MODE_COMPREHENSIVE, 1);
+ int cube_idB = test_fieldstat_cube_create(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1);
struct fieldstat *target = fieldstat_fork(master);
- fieldstat_destroy_cube(master, cube_idA);
+ fieldstat_cube_destroy(master, cube_idA);
fieldstat_calibrate(master, target);
diff --git a/test/test_write_json_file.cpp b/test/test_write_json_file.cpp
index ff3dedf..1dba94b 100644
--- a/test/test_write_json_file.cpp
+++ b/test/test_write_json_file.cpp
@@ -58,6 +58,14 @@ extern "C" {
return fse;
}
+int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k)
+{
+ assert(mode == SAMPLING_MODE_COMPREHENSIVE);
+ int ret = fieldstat_cube_create(instance, dimensions, n_dimensions);
+ fieldstat_cube_set_sampling(instance, ret, mode, k, 0);
+ return ret;
+}
+
static void write_hll(struct fieldstat *instance) {
struct field shared_tags[1];
@@ -66,7 +74,7 @@ static void write_hll(struct fieldstat *instance) {
shared_tags[0].value_longlong = 1;
const char *hll_name[] = {"external_ip", "internal_ip", "acc_ip"};
- int cube_id = fieldstat_create_cube(instance, shared_tags, 1,
+ int cube_id = test_fieldstat_cube_create(instance, shared_tags, 1,
SAMPLING_MODE_COMPREHENSIVE, 100);
for(unsigned int i = 0; i < sizeof(hll_name) / sizeof(hll_name[0]); i++)
@@ -107,7 +115,7 @@ void write_histogram(struct fieldstat *instance) {
"bye/udp", "oth_mtd/udp"};
- int cube_id = fieldstat_create_cube(instance, shared_tags, 2,
+ int cube_id = test_fieldstat_cube_create(instance, shared_tags, 2,
SAMPLING_MODE_COMPREHENSIVE, 100);
for(unsigned int i = 0; i < sizeof(hist_names)/sizeof(hist_names[0]); i++)
@@ -185,7 +193,7 @@ void write_table(struct fieldstat *instance) {
cell_tags.type = FIELD_VALUE_CSTRING;
cell_tags.value_str = "true";
- int cube_id = fieldstat_create_cube(instance, shared_tags, 2,
+ int cube_id = test_fieldstat_cube_create(instance, shared_tags, 2,
SAMPLING_MODE_COMPREHENSIVE, 100);
int counter_id_0 = fieldstat_register_counter(instance, cube_id, "T_success_log");