From da2b236902f842903bd7643e824454eff286a15d Mon Sep 17 00:00:00 2001 From: chenzizhan Date: Thu, 4 Jul 2024 15:05:36 +0800 Subject: move cube manager to cube.c; cube manager is now a bidict --- include/fieldstat/fieldstat.h | 9 - src/cube.c | 259 +++++++++++++++++- src/cube.h | 17 ++ src/exporter/cjson_exporter.c | 42 +-- src/fieldstat.c | 550 +++++++-------------------------------- src/tags/heavy_keeper.c | 17 +- src/tags/tag_map.h | 2 +- test/test_exporter_json.cpp | 8 - test/test_register_and_reset.cpp | 67 ++--- 9 files changed, 384 insertions(+), 587 deletions(-) diff --git a/include/fieldstat/fieldstat.h b/include/fieldstat/fieldstat.h index a6101a8..93ac2b8 100644 --- a/include/fieldstat/fieldstat.h +++ b/include/fieldstat/fieldstat.h @@ -84,11 +84,6 @@ int fieldstat_cube_set_primary_metric(struct fieldstat *instance, int cube_id, i */ int fieldstat_destroy_cube(struct fieldstat *instance, int cube_id); -/* - * @brief get the cube_version of the cube of cube_id. - * @return cube_version if success. FS_ERR_NULL_HANDLER or FS_ERR_INVALID_CUBE_ID if fail. -*/ -long long fieldstat_get_cube_version(const struct fieldstat *instance, int cube_id); /* * @brief add a metric to the cube of cube_id. One metric may be associated with different cells. * @param metric_name: name of the metric. Cannot be NULL. Must be unique. @@ -158,10 +153,6 @@ int fieldstat_hist_record(struct fieldstat *instance, int cube_id, int metric_id Note that the cell record won't be deleted at once, they just seem to be deleted. The cell record will be deleted when they are not used since the last reset and until the next reset. */ void fieldstat_reset(struct fieldstat *instance); -/* - version is increased by 1 when we call fieldstat_reset. -*/ -unsigned long fieldstat_get_version(const struct fieldstat *instance); /* @brief Merge the instance. The registered cubes and metrics are merged even if there are no cells added. diff --git a/src/cube.c b/src/cube.c index bbc91a4..22d3df2 100644 --- a/src/cube.c +++ b/src/cube.c @@ -9,14 +9,31 @@ #include "metric.h" #include "heavy_keeper.h" #include "tag_map.h" +#include "uthash.h" #define DEFAULT_N_METRIC 32 +#define DEFAULT_N_CUBE 64 struct exdata_new_args { const struct fieldstat_tag *tags; size_t n_tags; }; +struct cube_manager_item { + char *key; + size_t key_len; + int id; + UT_hash_handle hh; +}; + +struct cube_manager { + struct cube_manager_item *head; + + struct cube **cube; + size_t cube_cnt; + size_t cube_size; +}; + struct cell { struct metric **metrics; size_t max_n_metric; @@ -72,6 +89,220 @@ static void fieldstat_free_tag_array(struct fieldstat_tag *tags, size_t n_tags) free(tags); } +void add_cube_to_position(struct cube_manager *instance, struct cube *cube, int id) +{ + if (id >= instance->cube_size) { + struct cube **old_cube_arr = instance->cube; + instance->cube = calloc(instance->cube_size * 2, sizeof(struct cube *)); + memcpy(instance->cube, old_cube_arr, sizeof(struct cube *) * instance->cube_size); + free(old_cube_arr); + + instance->cube_size *= 2; + } + instance->cube[id] = cube; + + if (id >= instance->cube_cnt) { + instance->cube_cnt = id + 1; + } +} + +void cube_manager_free(struct cube_manager *pthis) { + struct cube_manager_item *node = NULL; + struct cube_manager_item *tmp = NULL; + struct cube_manager_item *head = pthis->head; + HASH_ITER(hh, head, node, tmp) { + HASH_DEL(head, node); + free(node->key); + free(node); + } + + for (int i = 0; i < pthis->cube_cnt; i++) { + if (pthis->cube[i] != NULL) { + cube_free(pthis->cube[i]); + } + } + + free(pthis->cube); + free(pthis); +} + +struct cube_manager *cube_manager_new() { + struct cube_manager *pthis = (struct cube_manager *)malloc(sizeof(struct cube_manager)); + pthis->head = NULL; + + pthis->cube = (struct cube **)calloc(DEFAULT_N_CUBE, sizeof(struct cube *)); + pthis->cube_cnt = 0; + pthis->cube_size = DEFAULT_N_CUBE; + return pthis; +} + +int cube_manager_add(struct cube_manager *pthis, struct cube *cube) +{ + char *key; + size_t key_len; + build_dynamic_cell_key(cube->cube_identifier, cube->n_shared_tags, &key, &key_len); + + struct cube_manager_item *node = NULL; + HASH_FIND(hh, pthis->head, key, key_len, node); + if (node != NULL) { + free(key); + return -1; + } + + int id = 0; + for ( ;id < pthis->cube_cnt; id++) { + if (pthis->cube[id] == NULL) { + break; + } + } + + node = (struct cube_manager_item *)malloc(sizeof(struct cube_manager_item)); + node->key = key; + node->key_len = key_len; + node->id = id; + HASH_ADD_KEYPTR(hh, pthis->head, node->key, key_len, node); + + add_cube_to_position(pthis, cube, id); + + return id; +} + +void cube_manager_delete_by_key(struct cube_manager *pthis, const char *key, size_t key_len) +{ + struct cube_manager_item *node = NULL; + HASH_FIND(hh, pthis->head, key, key_len, node); + if (node == NULL) { + return; + } + + int id = node->id; + cube_free(pthis->cube[id]); + pthis->cube[id] = NULL; + if (id == pthis->cube_cnt - 1) { + pthis->cube_cnt--; + } + + HASH_DEL(pthis->head, node); + free(node->key); + free(node); +} + +void cube_manager_delete(struct cube_manager *pthis, struct cube *cube) +{ + char *key; + size_t key_len; + build_dynamic_cell_key(cube->cube_identifier, cube->n_shared_tags, &key, &key_len); + + cube_manager_delete_by_key(pthis, key, key_len); + free(key); +} + +int cube_manager_find(const struct cube_manager *pthis, const struct fieldstat_tag *identifier, size_t n_tag) +{ + char *key; + size_t key_len; + build_dynamic_cell_key(identifier, n_tag, &key, &key_len); + + struct cube_manager_item *node = NULL; + HASH_FIND(hh, pthis->head, key, key_len, node); + free(key); + if (node == NULL) { + return -1; + } else { + return node->id; + } +} + +struct cube *cube_manager_get_cube_by_id(const struct cube_manager *manager, int cube_id) { + if (cube_id < 0 || cube_id >= manager->cube_size) { + return NULL; + } + return manager->cube[cube_id]; +} + +void cube_manager_list(const struct cube_manager *pthis, int **cube_ids, int *n_cube) { + int all_available_cube_count = 0; + + int *tmp_ids = (int *)malloc(sizeof(int) * pthis->cube_cnt); + for (int i = 0; i < pthis->cube_cnt; i++) { + if (pthis->cube[i] != NULL) { + tmp_ids[all_available_cube_count++] = i; + } + } + if (all_available_cube_count == 0) { + free(tmp_ids); + *cube_ids = NULL; + *n_cube = 0; + return; + } + + *cube_ids = tmp_ids; + *n_cube = all_available_cube_count; +} + +void cube_manager_calibrate(struct cube_manager *pthis, const struct cube_manager *master) +{ + struct cube_manager_item *node = NULL; + struct cube_manager_item *tmp = NULL; + + // exist in self but not in master + HASH_ITER(hh, pthis->head, node, tmp) { + struct cube_manager_item *node_in_master = NULL; + HASH_FIND(hh, master->head, node->key, node->key_len, node_in_master); + + if (node_in_master == NULL) { + cube_manager_delete_by_key(pthis, node->key, node->key_len); + } + } + + // exist in master but not in self + HASH_ITER(hh, master->head, node, tmp) { + struct cube_manager_item *node_in_self = NULL; + HASH_FIND(hh, pthis->head, node->key, node->key_len, node_in_self); + + if (node_in_self == NULL) { + int cube_id = node->id; + cube_manager_add(pthis, cube_fork(master->cube[cube_id])); + } + } +} + +struct cube_manager *cube_manager_fork(const struct cube_manager *src) +{ + struct cube_manager *pthis = cube_manager_new(); + struct cube_manager_item *node = NULL; + struct cube_manager_item *tmp = NULL; + HASH_ITER(hh, src->head, node, tmp) { + cube_manager_add(pthis, cube_fork(src->cube[node->id])); + } + return pthis; +} + +void cube_manager_merge(struct cube_manager *dest, const struct cube_manager *src) +{ + struct cube_manager_item *node = NULL; + struct cube_manager_item *tmp = NULL; + HASH_ITER(hh, src->head, node, tmp) { + struct cube_manager_item *node_in_dest = NULL; + HASH_FIND(hh, dest->head, node->key, node->key_len, node_in_dest); + + if (node_in_dest == NULL) { + cube_manager_add(dest, cube_copy(src->cube[node->id])); + } else { + cube_merge(dest->cube[node_in_dest->id], src->cube[node->id]); + } + } +} + +void cube_manager_reset(struct cube_manager *pthis) +{ + for (int i = 0; i < pthis->cube_cnt; i++) { + if (pthis->cube[i] == NULL) { + continue; + } + cube_reset(pthis->cube[i]); + } +} struct metric *find_metric_in_cell(const struct cell *cell, int metric_id) { @@ -195,7 +426,7 @@ void *exdata_copy_i(void *exdata) { return cell_copy((struct cell *)exdata); } -struct cube *fieldstat_cube_info_init(const struct fieldstat_tag *shared_tags, size_t n_tag, enum sampling_mode mode, size_t max_n_cell) +struct cube *cube_info_new(const struct fieldstat_tag *shared_tags, size_t n_tag, enum sampling_mode mode, size_t max_n_cell) { struct cube *cube = calloc(1, sizeof(struct cube)); cube->sampling_mode = mode; @@ -216,7 +447,7 @@ struct cube *fieldstat_cube_info_init(const struct fieldstat_tag *shared_tags, s struct cube *cube_new(const struct fieldstat_tag *shared_tags, size_t n_tag, enum sampling_mode mode, size_t max_n_cell) { - struct cube *cube = fieldstat_cube_info_init(shared_tags, n_tag, mode, max_n_cell); + struct cube *cube = cube_info_new(shared_tags, n_tag, mode, max_n_cell); switch (mode) { @@ -267,7 +498,7 @@ void cube_set_primary_metric(struct cube *cube, int metric_id) { cube->primary_metric_id = metric_id; } -struct cell *find_or_add_exdata_comprehensive(struct tag_map *comprehensive, const char *key, size_t key_len, struct exdata_new_args *args) +struct cell *find_or_add_cell_comprehensive(struct tag_map *comprehensive, const char *key, size_t key_len, struct exdata_new_args *args) { struct cell *cell_data = tag_map_get0_exdata(comprehensive, key, key_len); if (cell_data == NULL) { @@ -280,7 +511,7 @@ struct cell *find_or_add_exdata_comprehensive(struct tag_map *comprehensive, con return cell_data; } -struct cell *find_or_add_exdata_none_primary_topk(struct heavy_keeper *topk, const char *key, size_t key_len, struct exdata_new_args *args) +struct cell *find_or_add_cell_none_primary_topk(struct heavy_keeper *topk, const char *key, size_t key_len, struct exdata_new_args *args) { struct cell *cell_data = heavy_keeper_get0_exdata(topk, key, key_len); if (cell_data == NULL) { @@ -292,7 +523,7 @@ struct cell *find_or_add_exdata_none_primary_topk(struct heavy_keeper *topk, con } return cell_data; } - +// TODO: 整个Switch case 改成 cube_get_cell int cube_histogram_record(struct cube *cube, const struct metric_manifest *manifest, const struct fieldstat_tag *tags, size_t n_tag, long long value) { char *tag_in_string; @@ -306,10 +537,10 @@ int cube_histogram_record(struct cube *cube, const struct metric_manifest *manif struct cell *cell_data = NULL; switch (cube->sampling_mode) { case SAMPLING_MODE_TOPK: - cell_data = find_or_add_exdata_none_primary_topk(cube->topk, tag_in_string, tag_len, &args); + cell_data = find_or_add_cell_none_primary_topk(cube->topk, tag_in_string, tag_len, &args); break; case SAMPLING_MODE_COMPREHENSIVE: - cell_data = find_or_add_exdata_comprehensive(cube->comprehensive, tag_in_string, tag_len, &args); + cell_data = find_or_add_cell_comprehensive(cube->comprehensive, tag_in_string, tag_len, &args); break; } free(tag_in_string); @@ -338,10 +569,10 @@ int cube_hll_add(struct cube *cube, const struct metric_manifest *manifest, cons struct cell *cell_data = NULL; switch (cube->sampling_mode) { case SAMPLING_MODE_TOPK: - cell_data = find_or_add_exdata_none_primary_topk(cube->topk, tag_in_string, tag_len, &args); + cell_data = find_or_add_cell_none_primary_topk(cube->topk, tag_in_string, tag_len, &args); break; case SAMPLING_MODE_COMPREHENSIVE: - cell_data = find_or_add_exdata_comprehensive(cube->comprehensive, tag_in_string, tag_len, &args); + cell_data = find_or_add_cell_comprehensive(cube->comprehensive, tag_in_string, tag_len, &args); break; } @@ -373,7 +604,7 @@ int cube_counter_incrby(struct cube *cube, const struct metric_manifest *manifes cell_data = heavy_keeper_get0_exdata(cube->topk, tag_in_string, tag_len); if (cell_data == NULL) { - int tmp_ret = heavy_keeper_add(cube->topk, tag_in_string, tag_len, 0, &args); + int tmp_ret = heavy_keeper_add(cube->topk, tag_in_string, tag_len, 0, &args); // TODO: 忘了提取函数 if (tmp_ret != 1) { free(tag_in_string); return FS_ERR_TOO_MANY_CELLS; @@ -399,7 +630,7 @@ int cube_counter_incrby(struct cube *cube, const struct metric_manifest *manifes } break; case SAMPLING_MODE_COMPREHENSIVE: - cell_data = find_or_add_exdata_comprehensive(cube->comprehensive, tag_in_string, tag_len, &args); + cell_data = find_or_add_cell_comprehensive(cube->comprehensive, tag_in_string, tag_len, &args); if (cell_data == NULL) { free(tag_in_string); return FS_ERR_TOO_MANY_CELLS; @@ -432,7 +663,7 @@ int cube_counter_set(struct cube *cube, const struct metric_manifest *manifest, { case SAMPLING_MODE_TOPK: { // TODO: 这个地方想办法拿到值以后进counter incrby 流程，重构 if (cube->primary_metric_id != metric_id) { - cell_data = find_or_add_exdata_none_primary_topk(cube->topk, tag_in_string, tag_len, &args); + cell_data = find_or_add_cell_none_primary_topk(cube->topk, tag_in_string, tag_len, &args); } else { long long current_count = 0; cell_data = heavy_keeper_get0_exdata(cube->topk, tag_in_string, tag_len); @@ -459,7 +690,7 @@ int cube_counter_set(struct cube *cube, const struct metric_manifest *manifest, } break;} case SAMPLING_MODE_COMPREHENSIVE: { - cell_data = find_or_add_exdata_comprehensive(cube->comprehensive, tag_in_string, tag_len, &args); + cell_data = find_or_add_cell_comprehensive(cube->comprehensive, tag_in_string, tag_len, &args); break;} default: assert(0); @@ -474,7 +705,7 @@ int cube_counter_set(struct cube *cube, const struct metric_manifest *manifest, struct cube *cube_copy(const struct cube *cube) { - struct cube *cube_dup = fieldstat_cube_info_init(cube->cube_identifier, cube->n_shared_tags, cube->sampling_mode, cube->max_n_cell); + struct cube *cube_dup = cube_info_new(cube->cube_identifier, cube->n_shared_tags, cube->sampling_mode, cube->max_n_cell); cube_dup->primary_metric_id = cube->primary_metric_id; switch (cube->sampling_mode) diff --git a/src/cube.h b/src/cube.h index 71b83f4..1ebcfbb 100644 --- a/src/cube.h +++ b/src/cube.h @@ -11,6 +11,7 @@ extern "C" #include "metric_manifest.h" struct cube; +struct cube_manager; // TODO 这个应该从fieldstat 移进来 struct cube *cube_new(const struct fieldstat_tag *shared_tags, size_t n_tag, enum sampling_mode mode, size_t max_n_cell); void cube_free(struct cube *cube); @@ -36,6 +37,22 @@ void cube_get_cells_used_by_metric(const struct cube *cube, const struct fieldst void cube_set_primary_metric(struct cube *cube, int metric_id); struct fieldstat_tag_list *cube_get_identifier(const struct cube *cube); +struct cube *cube_manager_get_cube_by_id(const struct cube_manager *manager, int cube_id); + +// the cube will be taken over by the manager, user do not free it. +int cube_manager_add(struct cube_manager *pthis, struct cube *cube); +void cube_manager_delete(struct cube_manager *pthis, struct cube *cube); // the cube will be freed by the manager +int cube_manager_find(const struct cube_manager *pthis, const struct fieldstat_tag *identifier, size_t n_tag); +struct cube_manager *cube_manager_new(); +void cube_manager_free(struct cube_manager *pthis); +void cube_manager_merge(struct cube_manager *dest, const struct cube_manager *src); + +void cube_manager_reset(struct cube_manager *pthis); +void cube_manager_calibrate(struct cube_manager *pthis, const struct cube_manager *master); +struct cube_manager *cube_manager_fork(const struct cube_manager *src); + +void cube_manager_list(const struct cube_manager *pthis, int **cube_ids, int *n_cube); + #ifdef __cplusplus } #endif \ No newline at end of file diff --git a/src/exporter/cjson_exporter.c b/src/exporter/cjson_exporter.c index 262ffe2..23e3503 100644 --- a/src/exporter/cjson_exporter.c +++ b/src/exporter/cjson_exporter.c @@ -100,10 +100,6 @@ struct counter_history { struct tag_metric_map *rec; long long ts; - unsigned long cell_version; - unsigned long *cube_version; - size_t n_cube; - char *exporter_name; struct fieldstat_tag_list *global_tag_list; }; @@ -209,8 +205,6 @@ void counter_history_free(struct counter_history *history) free(tag_node); } - free(history->cube_version); - free(history->exporter_name); if (history->global_tag_list != NULL) { @@ -287,22 +281,17 @@ struct fieldstat_tag_list *my_copy_fs_tag_list(const struct fieldstat_tag_list * bool counter_history_check_if_need_to_update(const struct fieldstat_json_exporter *exporter, const struct fieldstat *instance) { - struct counter_history *history = exporter->history; + const struct counter_history *history = exporter->history; if (history == NULL) { return false; // delta export disabled } - if (history->cube_version == NULL) { // first time + if (history->exporter_name == NULL) { // first time return true; } - unsigned long cur_cell_version = fieldstat_get_version(instance); const char *cur_exporter_name = exporter->name ? exporter->name : DEFAULT_EXPORTER_NAME; const struct fieldstat_tag_list *cur_global_tag_list = exporter->global_tag_list; - if (history->cell_version != cur_cell_version) { - return true; - } - if (strcmp(history->exporter_name, cur_exporter_name) != 0) { return true; } @@ -318,26 +307,11 @@ bool counter_history_check_if_need_to_update(const struct fieldstat_json_exporte return true; } - int *cube_ids = NULL; - int n_cube = 0; - fieldstat_get_cubes(instance, &cube_ids, &n_cube); - if (n_cube != history->n_cube) { - return true; - } - for (int i = 0; i < n_cube; i++) { - unsigned long cube_version = fieldstat_get_cube_version(instance, cube_ids[i]); - if (cube_version != history->cube_version[i]) { - return true; - } - } - free(cube_ids); - return false; } void counter_history_fill_version_info(struct counter_history *history, struct fieldstat_json_exporter *exporter, const struct fieldstat *instance) { - unsigned long cur_cell_version = fieldstat_get_version(instance); const char *cur_exporter_name = exporter->name ? exporter->name : DEFAULT_EXPORTER_NAME; const struct fieldstat_tag_list *cur_global_tag_list = exporter->global_tag_list; @@ -346,18 +320,6 @@ void counter_history_fill_version_info(struct counter_history *history, struct f if (cur_global_tag_list != NULL) { history->global_tag_list = my_copy_fs_tag_list(cur_global_tag_list); } - - history->cell_version = cur_cell_version; - - int *cube_ids = NULL; - int n_cube = 0; - fieldstat_get_cubes(instance, &cube_ids, &n_cube); - history->n_cube = n_cube; - history->cube_version = malloc(sizeof(unsigned long) * n_cube); - for (int i = 0; i < n_cube; i++) { - history->cube_version[i] = fieldstat_get_cube_version(instance, cube_ids[i]); - } - free(cube_ids); } void fieldstat_json_exporter_update_history(struct fieldstat_json_exporter *exporter, const struct fieldstat *instance) diff --git a/src/fieldstat.c b/src/fieldstat.c index 7a8a16e..dca5246 100644 --- a/src/fieldstat.c +++ b/src/fieldstat.c @@ -15,116 +15,17 @@ #include "metric_manifest.h" #define DEFAULT_N_METRIC 32 -#define DEFAULT_N_CUBE 64 -struct cube_manager_item { - char *key; - size_t key_len; - int cell_id; - UT_hash_handle hh; -}; - -struct cube_manager { - struct cube_manager_item *head; -}; - struct fieldstat { - struct cube **cube; - unsigned long *cube_version; // increase from 0 every time the cube is deleted - unsigned long cell_version; // increase from 0 every time fieldstat_reset is called - size_t max_n_cube; - - struct metric_manifest **metric_masters; - size_t max_n_metric_master; + struct metric_manifest **manifests; // TODO: 把那个哈希表再加回去 + size_t max_n_manifests; + // TODO: 就三个吧，还有一个count - struct cube_manager *shared_tag_cube_manager; + struct cube_manager *cube_manager; }; -void cube_manager_free(struct cube_manager *pthis) -{ - struct cube_manager_item *node = NULL; - struct cube_manager_item *tmp = NULL; - struct cube_manager_item *head = pthis->head; - HASH_ITER(hh, head, node, tmp) { - HASH_DEL(head, node); - free(node->key); - free(node); - } - free(pthis); -} - -struct cube_manager *cube_manager_new() -{ - struct cube_manager *pthis = (struct cube_manager *)malloc(sizeof(struct cube_manager)); - pthis->head = NULL; - return pthis; -} - -static char *key_dup(const char *key, size_t key_len) -{ - char *new_key = (char *)malloc(key_len); - memcpy(new_key, key, key_len); - return new_key; -} - -void cube_manager_add(struct cube_manager *pthis, const char *key, size_t key_len, int id) -{ - struct cube_manager_item *node = (struct cube_manager_item *)malloc(sizeof(struct cube_manager_item)); - node->key = key_dup(key, key_len); - node->key_len = key_len; - node->cell_id = id; - HASH_ADD_KEYPTR(hh, pthis->head, node->key, key_len, node); -} - -void cube_manager_delete(struct cube_manager *pthis, const char *key, size_t key_len) -{ - struct cube_manager_item *node = NULL; - HASH_FIND(hh, pthis->head, key, key_len, node); - if (node != NULL) { - HASH_DEL(pthis->head, node); - free(node->key); - free(node); - } -} - -int cube_manager_find(const struct cube_manager *pthis, const char *key, size_t key_len) -{ - struct cube_manager_item *node = NULL; - HASH_FIND(hh, pthis->head, key, key_len, node); - if (node == NULL) { - return -1; - } else { - return node->cell_id; - } -} - -void cube_manager_calibrate(struct cube_manager *pthis, const struct cube_manager *master) -{ - struct cube_manager_item *node = NULL; - struct cube_manager_item *tmp = NULL; - - // exist in self but not in master - HASH_ITER(hh, pthis->head, node, tmp) { - int cube_id = cube_manager_find(master, node->key, node->key_len); - if (cube_id == -1) { - HASH_DEL(pthis->head, node); - free(node->key); - free(node); - } else { - node->cell_id = cube_id; - } - } - - // exist in master but not in self - HASH_ITER(hh, master->head, node, tmp) { - int cube_id = cube_manager_find(pthis, node->key, node->key_len); - if (cube_id == -1) { - cube_manager_add(pthis, node->key, node->key_len, node->cell_id); - } - } -} union metric_parameter *construct_parameters(enum metric_type type, ...) { @@ -229,11 +130,11 @@ bool is_tag_array_same(const struct fieldstat_tag *tags1, const struct fieldstat bool is_metric_name_duplicate(const struct fieldstat *instance, const char *name) { - for (size_t i = 0; i < instance->max_n_metric_master; i++) { - if (instance->metric_masters[i] == NULL) { + for (size_t i = 0; i < instance->max_n_manifests; i++) { + if (instance->manifests[i] == NULL) { continue; } - if (strcmp(instance->metric_masters[i]->name, name) == 0) { + if (strcmp(instance->manifests[i]->name, name) == 0) { return true; } } @@ -247,14 +148,10 @@ struct fieldstat *fieldstat_new() { struct fieldstat *instance = calloc(1, sizeof(struct fieldstat)); - instance->max_n_cube = DEFAULT_N_CUBE; - instance->cube = calloc(instance->max_n_cube, sizeof(struct cube *)); - instance->cube_version = calloc(instance->max_n_cube, sizeof(unsigned long)); + instance->max_n_manifests = DEFAULT_N_METRIC; + instance->manifests = calloc(instance->max_n_manifests, sizeof(struct metric_manifest *)); - instance->max_n_metric_master = DEFAULT_N_METRIC; - instance->metric_masters = calloc(instance->max_n_metric_master, sizeof(struct metric_manifest *)); - - instance->shared_tag_cube_manager = cube_manager_new(); + instance->cube_manager = cube_manager_new(); return instance; } @@ -265,19 +162,15 @@ void fieldstat_free(struct fieldstat *instance) if (instance == NULL) { return; } - for (size_t i = 0; i < instance->max_n_cube; i++) { - fieldstat_cube_free(instance, i); - } - free(instance->cube); - free(instance->cube_version); - cube_manager_free(instance->shared_tag_cube_manager); - for (size_t i = 0; i < instance->max_n_metric_master; i++) { - if (instance->metric_masters[i] != NULL) { - metric_manifest_free(instance->metric_masters[i]); + cube_manager_free(instance->cube_manager); + + for (size_t i = 0; i < instance->max_n_manifests; i++) { + if (instance->manifests[i] != NULL) { + metric_manifest_free(instance->manifests[i]); } } - free(instance->metric_masters); + free(instance->manifests); free(instance); } @@ -287,72 +180,19 @@ void fieldstat_reset(struct fieldstat *instance) if (instance == NULL) { return; } - for (size_t i = 0; i < instance->max_n_cube; i++) { - struct cube *cube = instance->cube[i]; - if (cube == NULL) { - continue; - } - - cube_reset(cube); - } - instance->cell_version++; -} - -unsigned long fieldstat_get_version(const struct fieldstat *instance) -{ - if (instance == NULL) { - return 0; - } - return instance->cell_version; -} - -void get_cube_key(const struct cube *cube, char **key, size_t *key_len) -{ - struct fieldstat_tag_list *shared_tag = cube_get_identifier(cube); - build_dynamic_cell_key(shared_tag->tag, shared_tag->n_tag, key, key_len); - fieldstat_tag_list_arr_free(shared_tag, 1); + cube_manager_reset(instance->cube_manager); } int fieldstat_destroy_cube(struct fieldstat *instance, int cube_id) { - if (instance == NULL) { - return FS_ERR_NULL_HANDLER; - } - if (cube_id < 0 || cube_id >= instance->max_n_cube) { - return FS_ERR_INVALID_CUBE_ID; - } - if (instance->cube[cube_id] == NULL) { + struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + if (cube == NULL) { return FS_ERR_INVALID_CUBE_ID; } - const struct cube *cube = instance->cube[cube_id]; - char *shared_tag_key = NULL; - size_t shared_tag_key_len = 0; - get_cube_key(cube, &shared_tag_key, &shared_tag_key_len); - cube_manager_delete(instance->shared_tag_cube_manager, shared_tag_key, shared_tag_key_len); - - fieldstat_cube_free(instance, cube_id); + cube_manager_delete(instance->cube_manager, cube); - instance->cube[cube_id] = NULL; - instance->cube_version[cube_id]++; - - free(shared_tag_key); - return 0; -} - -long long fieldstat_get_cube_version(const struct fieldstat *instance, int cube_id) -{ - if (instance == NULL) { - return FS_ERR_NULL_HANDLER; - } - if (cube_id < 0 || cube_id >= instance->max_n_cube) { - return FS_ERR_INVALID_CUBE_ID; - } - if (instance->cube[cube_id] == NULL) { - return FS_ERR_INVALID_CUBE_ID; - } - - return instance->cube_version[cube_id]; + return FS_OK; } /* -------------------------------------------------------------------------- */ @@ -371,40 +211,6 @@ void fieldstat_free_tag_array(struct fieldstat_tag *tags, size_t n_tags) free(tags); } -void add_cube_to_position(struct fieldstat *instance, struct cube *cube, int cube_id) -{ - if (cube_id >= instance->max_n_cube) { - struct cube **old_cube_arr = instance->cube; - instance->cube = calloc(instance->max_n_cube * 2, sizeof(struct cube *)); - memcpy(instance->cube, old_cube_arr, sizeof(struct cube *) * instance->max_n_cube); - free(old_cube_arr); - - unsigned long *old_ver_arr = instance->cube_version; - instance->cube_version = calloc(instance->max_n_cube * 2, sizeof(unsigned long)); - memcpy(instance->cube_version, old_ver_arr, sizeof(unsigned long) * instance->max_n_cube); - free(old_ver_arr); - - instance->max_n_cube *= 2; - } - instance->cube[cube_id] = cube; -} - -int fieldstat_append_cube_to_instance(struct fieldstat *instance, struct cube *cube) -{ - for (int i = 0; i < instance->max_n_cube; i++) { - if (instance->cube[i] == NULL) { - instance->cube[i] = cube; - return i; - } - } - - int cube_id = instance->max_n_cube; - add_cube_to_position(instance, cube, cube_id); - - return cube_id; -} - - int fieldstat_create_cube(struct fieldstat *instance, const struct fieldstat_tag *shared_tags, size_t n_tag, enum sampling_mode mode, size_t max_n_cell) { if (instance == NULL) { @@ -422,39 +228,14 @@ int fieldstat_create_cube(struct fieldstat *instance, const struct fieldstat_tag max_n_cell = INT32_MAX; } - char *shared_tag_key = NULL; - size_t shared_tag_key_len = 0; - build_dynamic_cell_key(shared_tags, n_tag, &shared_tag_key, &shared_tag_key_len); - int ret = cube_manager_find(instance->shared_tag_cube_manager, shared_tag_key, shared_tag_key_len); - if (ret != -1) { - free(shared_tag_key); + struct cube *cube = cube_new(shared_tags, n_tag, mode, max_n_cell); + int ret = cube_manager_add(instance->cube_manager, cube); + if (ret < 0) { + cube_free(cube); return FS_ERR_INVALID_KEY; } - struct cube *cube = cube_new(shared_tags, n_tag, mode, max_n_cell); - - int cube_id = fieldstat_append_cube_to_instance(instance, cube); - cube_manager_add(instance->shared_tag_cube_manager, shared_tag_key, shared_tag_key_len, cube_id); - free(shared_tag_key); - - return cube_id; -} - -void fieldstat_cube_free(struct fieldstat *instance, int cube_id) -{ - struct cube *cube = instance->cube[cube_id]; - if (cube == NULL) { - return; - } - - char *shared_tag_key = NULL; - size_t shared_tag_key_len = 0; - get_cube_key(cube, &shared_tag_key, &shared_tag_key_len); - cube_manager_delete(instance->shared_tag_cube_manager, shared_tag_key, shared_tag_key_len); - free(shared_tag_key); - - cube_free(cube); - instance->cube[cube_id] = NULL; + return ret; //ret is the cube_id } int fieldstat_cube_set_primary_metric(struct fieldstat *instance, int cube_id, int metric_id) @@ -462,20 +243,18 @@ int fieldstat_cube_set_primary_metric(struct fieldstat *instance, int cube_id, i if (instance == NULL) { return FS_ERR_NULL_HANDLER; } - if (cube_id < 0 || cube_id >= instance->max_n_cube) { - return FS_ERR_INVALID_CUBE_ID; - } - struct cube *cube = instance->cube[cube_id]; + + struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); if (cube == NULL) { return FS_ERR_INVALID_CUBE_ID; } - if (metric_id < 0 || metric_id >= instance->max_n_metric_master) { + if (metric_id < 0 || metric_id >= instance->max_n_manifests) { return FS_ERR_INVALID_METRIC_ID; } - if (instance->metric_masters[metric_id] == NULL) { + if (instance->manifests[metric_id] == NULL) { return FS_ERR_INVALID_METRIC_ID; } - if (instance->metric_masters[metric_id]->type != METRIC_TYPE_COUNTER) { + if (instance->manifests[metric_id]->type != METRIC_TYPE_COUNTER) { return FS_ERR_INVALID_PARAM; } @@ -490,20 +269,20 @@ int fieldstat_cube_set_primary_metric(struct fieldstat *instance, int cube_id, i void add_manifest_to_instance(struct fieldstat *instance, const struct metric_manifest *manifest, int metric_id) { - if (metric_id >= instance->max_n_metric_master) { - instance->metric_masters = realloc(instance->metric_masters, sizeof(struct metric_manifest *) * instance->max_n_metric_master * 2); - memset(instance->metric_masters + instance->max_n_metric_master, 0, sizeof(struct metric_manifest *) * (instance->max_n_metric_master)); - instance->max_n_metric_master *= 2; + if (metric_id >= instance->max_n_manifests) { + instance->manifests = realloc(instance->manifests, sizeof(struct metric_manifest *) * instance->max_n_manifests * 2); + memset(instance->manifests + instance->max_n_manifests, 0, sizeof(struct metric_manifest *) * (instance->max_n_manifests)); + instance->max_n_manifests *= 2; } - instance->metric_masters[metric_id] = (struct metric_manifest *)manifest; + instance->manifests[metric_id] = (struct metric_manifest *)manifest; } static int append_manifest_to_instance(struct fieldstat *instance, const struct metric_manifest *metric) { int metric_id = 0; - for ( ;metric_id < instance->max_n_metric_master; metric_id++) { - if (instance->metric_masters[metric_id] == NULL) { + for ( ;metric_id < instance->max_n_manifests; metric_id++) { + if (instance->manifests[metric_id] == NULL) { break; } } @@ -603,17 +382,14 @@ int check_before_add(const struct fieldstat *instance, int cube_id, int metric_i return FS_ERR_NULL_HANDLER; } - if (cube_id < 0 || cube_id >= instance->max_n_cube) { - return FS_ERR_INVALID_CUBE_ID; - } - if (instance->cube[cube_id] == NULL) { + if (cube_manager_get_cube_by_id(instance->cube_manager, cube_id) == NULL) { return FS_ERR_INVALID_CUBE_ID; } - if (metric_id < 0 || metric_id >= instance->max_n_metric_master) { + if (metric_id < 0 || metric_id >= instance->max_n_manifests) { return FS_ERR_INVALID_METRIC_ID; } - const struct metric_manifest *metric = instance->metric_masters[metric_id]; + const struct metric_manifest *metric = instance->manifests[metric_id]; if (metric == NULL || metric->type != type) { return FS_ERR_INVALID_METRIC_ID; } @@ -628,8 +404,8 @@ int fieldstat_counter_incrby(struct fieldstat *instance, int cube_id, int metric return ret; } - struct cube *cube = instance->cube[cube_id]; - const struct metric_manifest *manifest = instance->metric_masters[metric_id]; + struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + const struct metric_manifest *manifest = instance->manifests[metric_id]; return cube_counter_incrby(cube, manifest, tags, n_tag, increment); @@ -641,8 +417,8 @@ int fieldstat_counter_set(struct fieldstat *instance, int cube_id, int metric_id if (ret != FS_OK) { return ret; } - struct cube *cube = instance->cube[cube_id]; - const struct metric_manifest *manifest = instance->metric_masters[metric_id]; + struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + const struct metric_manifest *manifest = instance->manifests[metric_id]; return cube_counter_set(cube, manifest, tags, n_tag, value); } @@ -654,8 +430,8 @@ int fieldstat_hll_add(struct fieldstat *instance, int cube_id, int metric_id, co return ret; } - struct cube *cube = instance->cube[cube_id]; - const struct metric_manifest *manifest = instance->metric_masters[metric_id]; + struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + const struct metric_manifest *manifest = instance->manifests[metric_id]; return cube_hll_add(cube, manifest, tags, n_tag, key, key_len); } @@ -667,8 +443,8 @@ int fieldstat_hist_record(struct fieldstat *instance, int cube_id, int metric_id return ret; } - struct cube *cube = instance->cube[cube_id]; - const struct metric_manifest *manifest = instance->metric_masters[metric_id]; + struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + const struct metric_manifest *manifest = instance->manifests[metric_id]; return cube_histogram_record(cube, manifest, tags, n_tag, value); } @@ -685,10 +461,10 @@ int fieldstat_merge(struct fieldstat *instance, struct fieldstat *src) return FS_ERR_NULL_HANDLER; } - int metric_len_src = src->max_n_metric_master; + int metric_len_src = src->max_n_manifests; for (int i = 0; i < metric_len_src; i++) { - const struct metric_manifest *metric_src = src->metric_masters[i]; - const struct metric_manifest *metric_dst = instance->metric_masters[i]; + const struct metric_manifest *metric_src = src->manifests[i]; + const struct metric_manifest *metric_dst = instance->manifests[i]; if (metric_src == NULL || metric_dst == NULL) { break; } @@ -700,39 +476,16 @@ int fieldstat_merge(struct fieldstat *instance, struct fieldstat *src) } } for (int i = 0; i < metric_len_src; i++) { - if (instance->metric_masters[i] != NULL || src->metric_masters[i] == NULL) { + if (instance->manifests[i] != NULL || src->manifests[i] == NULL) { continue; } - const struct metric_manifest *metric_src = src->metric_masters[i]; + const struct metric_manifest *metric_src = src->manifests[i]; append_manifest_to_instance(instance, metric_manifest_copy(metric_src)); } - struct cube_manager *tag_cube_id_map = instance->shared_tag_cube_manager; - int ret = 0; - for (int i = 0; i < src->max_n_cube; i++) { - const struct cube *cube_src = src->cube[i]; - if (cube_src == NULL) { - continue; - } - - struct fieldstat_tag_list *shared_tag_list = cube_get_identifier(cube_src); - char *shared_tag_key_src = NULL; - size_t shared_tag_key_len = 0; - build_dynamic_cell_key(shared_tag_list->tag, shared_tag_list->n_tag, &shared_tag_key_src, &shared_tag_key_len); - int cube_id_dest = cube_manager_find(tag_cube_id_map, shared_tag_key_src, shared_tag_key_len); - if (cube_id_dest == -1) { - struct cube *copied_cube = cube_copy(cube_src); - cube_id_dest = fieldstat_append_cube_to_instance(instance, copied_cube); - cube_manager_add(tag_cube_id_map, shared_tag_key_src, shared_tag_key_len, cube_id_dest); - } else { - struct cube *cube_dst = instance->cube[cube_id_dest]; - cube_merge(cube_dst, cube_src); - } - fieldstat_tag_list_arr_free(shared_tag_list, 1); - free(shared_tag_key_src); - } + cube_manager_merge(instance->cube_manager, src->cube_manager); - return ret; + return FS_OK; } struct fieldstat *fieldstat_fork(const struct fieldstat *instance) @@ -741,31 +494,13 @@ struct fieldstat *fieldstat_fork(const struct fieldstat *instance) return NULL; } struct fieldstat *new_instance = calloc(1, sizeof(struct fieldstat)); - new_instance->shared_tag_cube_manager = cube_manager_new(); + new_instance->cube_manager = cube_manager_fork(instance->cube_manager); - new_instance->max_n_cube = instance->max_n_cube; - new_instance->cube = calloc(new_instance->max_n_cube, sizeof(struct cube *)); - for (size_t i = 0; i < new_instance->max_n_cube; i++) { - const struct cube *cube = instance->cube[i]; - if (cube == NULL) { - continue; - } - new_instance->cube[i] = cube_fork(cube); - - char *shared_tag_key = NULL; - size_t shared_tag_key_len = 0; - get_cube_key(cube, &shared_tag_key, &shared_tag_key_len); - cube_manager_add(new_instance->shared_tag_cube_manager, shared_tag_key, shared_tag_key_len, i); - free(shared_tag_key); - } - new_instance->cube_version = calloc(new_instance->max_n_cube, sizeof(unsigned long)); - memcpy(new_instance->cube_version, instance->cube_version, sizeof(unsigned long) * new_instance->max_n_cube); - - new_instance->metric_masters = calloc(instance->max_n_metric_master, sizeof(struct metric_manifest *)); - new_instance->max_n_metric_master = instance->max_n_metric_master; - for (size_t i = 0; i < instance->max_n_metric_master; i++) { - if (instance->metric_masters[i] != NULL) { - new_instance->metric_masters[i] = metric_manifest_copy(instance->metric_masters[i]); + new_instance->manifests = calloc(instance->max_n_manifests, sizeof(struct metric_manifest *)); + new_instance->max_n_manifests = instance->max_n_manifests; + for (size_t i = 0; i < instance->max_n_manifests; i++) { + if (instance->manifests[i] != NULL) { + new_instance->manifests[i] = metric_manifest_copy(instance->manifests[i]); } } @@ -774,22 +509,22 @@ struct fieldstat *fieldstat_fork(const struct fieldstat *instance) void calibrate_metrics_in_instance(const struct fieldstat *master, struct fieldstat *replica) { - if (replica->max_n_metric_master < master->max_n_metric_master) { - replica->metric_masters = (struct metric_manifest **)realloc(replica->metric_masters, sizeof(struct metric_manifest *) * master->max_n_metric_master); - memset(replica->metric_masters + replica->max_n_metric_master, 0, sizeof(struct metric_manifest *) * (master->max_n_metric_master - replica->max_n_metric_master)); - replica->max_n_metric_master = master->max_n_metric_master; + if (replica->max_n_manifests < master->max_n_manifests) { + replica->manifests = (struct metric_manifest **)realloc(replica->manifests, sizeof(struct metric_manifest *) * master->max_n_manifests); + memset(replica->manifests + replica->max_n_manifests, 0, sizeof(struct metric_manifest *) * (master->max_n_manifests - replica->max_n_manifests)); + replica->max_n_manifests = master->max_n_manifests; } - size_t longer_arr_len = master->max_n_metric_master > replica->max_n_metric_master ? master->max_n_metric_master : replica->max_n_metric_master; + size_t longer_arr_len = master->max_n_manifests > replica->max_n_manifests ? master->max_n_manifests : replica->max_n_manifests; for (size_t i = 0; i < longer_arr_len; i++) { - const struct metric_manifest *metric_master = i >= master->max_n_metric_master ? NULL : master->metric_masters[i]; - struct metric_manifest *metric_target = i >= replica->max_n_metric_master ? NULL : replica->metric_masters[i]; + const struct metric_manifest *metric_master = i >= master->max_n_manifests ? NULL : master->manifests[i]; + struct metric_manifest *metric_target = i >= replica->max_n_manifests ? NULL : replica->manifests[i]; if (metric_master == NULL && metric_target == NULL) { continue; } if (metric_master == NULL && metric_target != NULL) { metric_manifest_free(metric_target); - replica->metric_masters[i] = NULL; + replica->manifests[i] = NULL; continue; } if (metric_master != NULL && metric_target == NULL) { @@ -811,56 +546,7 @@ void calibrate_metrics_in_instance(const struct fieldstat *master, struct fields int fieldstat_calibrate(const struct fieldstat *master, struct fieldstat *replica) { - if (master == NULL || replica == NULL) { - return FS_ERR_NULL_HANDLER; - } - - if (replica->max_n_cube < master->max_n_cube) { - replica->cube = (struct cube **)realloc(replica->cube, sizeof(struct cube *) * master->max_n_cube); - memset(replica->cube + replica->max_n_cube, 0, sizeof(struct cube *) * (master->max_n_cube - replica->max_n_cube)); - replica->cube_version = (unsigned long *)realloc(replica->cube_version, sizeof(unsigned long) * master->max_n_cube); - memset(replica->cube_version + replica->max_n_cube, 0, sizeof(unsigned long) * (master->max_n_cube - replica->max_n_cube)); - replica->max_n_cube = master->max_n_cube; - } - - size_t len_master = master->max_n_cube; - size_t len_replica = replica->max_n_cube; - size_t longer_arr_len = len_master > len_replica ? len_master : len_replica; - for (size_t i = 0; i < longer_arr_len; i++) { - const struct cube *cube_master = i >= len_master ? NULL : master->cube[i]; - struct cube *cube_target = i >= len_replica ? NULL : replica->cube[i]; - - if (cube_master == NULL && cube_target == NULL) { - continue; - } - if (cube_master == NULL && cube_target != NULL) { - cube_free(cube_target); - replica->cube[i] = NULL; - continue; - } - if (cube_master != NULL && cube_target == NULL) { - struct cube *cube_dup = cube_fork(cube_master); - add_cube_to_position(replica, cube_dup, i); - continue; - } - - struct fieldstat_tag_list *cube_master_identifier = cube_get_identifier(cube_master); - struct fieldstat_tag_list *cube_target_identifier = cube_get_identifier(cube_target); - bool is_same = cube_master_identifier->n_tag == cube_target_identifier->n_tag && is_tag_array_same(cube_master_identifier->tag, cube_target_identifier->tag, cube_master_identifier->n_tag); - fieldstat_tag_list_arr_free(cube_master_identifier, 1); - fieldstat_tag_list_arr_free(cube_target_identifier, 1); - if (master->cube_version[i] == replica->cube_version[i] && is_same) { - continue; - } - - cube_free(cube_target); - struct cube *cube_dup = cube_fork(cube_master); - add_cube_to_position(replica, cube_dup, i); - } - - memcpy(replica->cube_version, master->cube_version, sizeof(unsigned long) * master->max_n_cube); - - cube_manager_calibrate(replica->shared_tag_cube_manager, master->shared_tag_cube_manager); + cube_manager_calibrate(replica->cube_manager, master->cube_manager); calibrate_metrics_in_instance(master, replica); return FS_OK; @@ -871,38 +557,16 @@ int fieldstat_calibrate(const struct fieldstat *master, struct fieldstat *replic /* -------------------------------------------------------------------------- */ void fieldstat_get_cubes(const struct fieldstat *instance, int **cube_ids, int *n_cube) { - int all_available_cube_count = 0; - - int *tmp_ids = (int *)malloc(sizeof(int) * instance->max_n_cube); - for (int i = 0; i < instance->max_n_cube; i++) { - const struct cube *tmp_cube = instance->cube[i]; - if (tmp_cube == NULL) { - continue; - } - tmp_ids[all_available_cube_count] = i; - all_available_cube_count ++; - } - if (all_available_cube_count == 0) { - free(tmp_ids); - *cube_ids = NULL; - *n_cube = 0; - return; - } - - *cube_ids = (int *)malloc(sizeof(int) * all_available_cube_count); - memcpy(*cube_ids, tmp_ids, sizeof(int) * all_available_cube_count); - *n_cube = all_available_cube_count; - - free(tmp_ids); + cube_manager_list(instance->cube_manager, cube_ids, n_cube); } void fieldstat_get_metrics(const struct fieldstat *instance, int **metric_id_out, size_t *n_metric) { - int *tmp_ids = (int *)malloc(sizeof(int) * instance->max_n_metric_master); + int *tmp_ids = (int *)malloc(sizeof(int) * instance->max_n_manifests); *metric_id_out = tmp_ids; int cnt = 0; - for (int i = 0; i < instance->max_n_metric_master; i++) { - if (instance->metric_masters[i] != NULL) { + for (int i = 0; i < instance->max_n_manifests; i++) { + if (instance->manifests[i] != NULL) { tmp_ids[cnt] = i; cnt ++; } @@ -916,10 +580,7 @@ void fieldstat_get_metrics(const struct fieldstat *instance, int **metric_id_out struct fieldstat_tag_list *fieldstat_get_shared_tags(const struct fieldstat *instance, int cube_id) { - if (instance == NULL || cube_id >= instance->max_n_cube || cube_id < 0) { - return NULL; - } - const struct cube *cube = instance->cube[cube_id]; + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); if (cube == NULL) { return NULL; } @@ -929,10 +590,7 @@ struct fieldstat_tag_list *fieldstat_get_shared_tags(const struct fieldstat *ins int fieldstat_counter_get(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tags, long long *value) { - if (cube_id < 0 || cube_id >= instance->max_n_cube) { - return FS_ERR_INVALID_CUBE_ID; - } - const struct cube *cube = instance->cube[cube_id]; + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); if (cube == NULL) { return FS_ERR_INVALID_CUBE_ID; } @@ -942,10 +600,7 @@ int fieldstat_counter_get(const struct fieldstat *instance, int cube_id, int met int fieldstat_hll_get(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tags, double *value) { - if (cube_id < 0 || cube_id >= instance->max_n_cube) { - return FS_ERR_INVALID_CUBE_ID; - } - const struct cube *cube = instance->cube[cube_id]; + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); if (cube == NULL) { return FS_ERR_INVALID_CUBE_ID; } @@ -957,10 +612,7 @@ int fieldstat_hll_get(const struct fieldstat *instance, int cube_id, int metric_ long long fieldstat_hist_value_at_percentile(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tags, double percentile) { - if (cube_id < 0 || cube_id >= instance->max_n_cube) { - return FS_ERR_INVALID_CUBE_ID; - } - const struct cube *cube = instance->cube[cube_id]; + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); if (cube == NULL) { return FS_ERR_INVALID_CUBE_ID; } @@ -975,10 +627,7 @@ long long fieldstat_hist_value_at_percentile(const struct fieldstat *instance, i long long fieldstat_hist_count_le_value(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tags, long long value) { - if (cube_id < 0 || cube_id >= instance->max_n_cube) { - return FS_ERR_INVALID_CUBE_ID; - } - const struct cube *cube = instance->cube[cube_id]; + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); if (cube == NULL) { return FS_ERR_INVALID_CUBE_ID; } @@ -992,13 +641,10 @@ long long fieldstat_hist_count_le_value(const struct fieldstat *instance, int cu void fieldstat_get_serialized_blob(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tags, char **blob, size_t *blob_size) { - *blob = NULL; - *blob_size = 0; - if (cube_id < 0 || cube_id >= instance->max_n_cube) { - return; - } - const struct cube *cube = instance->cube[cube_id]; + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); if (cube == NULL) { + *blob = NULL; + *blob_size = 0; return; } @@ -1018,10 +664,10 @@ void fieldstat_tag_list_arr_free(struct fieldstat_tag_list *tag_list, size_t n_c const char *fieldstat_get_metric_name(const struct fieldstat *instance, int metric_id) { - if (metric_id < 0 || metric_id >= instance->max_n_metric_master) { + if (metric_id < 0 || metric_id >= instance->max_n_manifests) { return NULL; } - const struct metric_manifest *metric = instance->metric_masters[metric_id]; + const struct metric_manifest *metric = instance->manifests[metric_id]; if (metric == NULL) { return NULL; } @@ -1031,10 +677,10 @@ const char *fieldstat_get_metric_name(const struct fieldstat *instance, int metr enum metric_type fieldstat_get_metric_type(const struct fieldstat *instance, int metric_id) { - if (instance == NULL || metric_id < 0 || metric_id >= instance->max_n_metric_master) { + if (instance == NULL || metric_id < 0 || metric_id >= instance->max_n_manifests) { return (enum metric_type)(-1); } - const struct metric_manifest *metric = instance->metric_masters[metric_id]; + const struct metric_manifest *metric = instance->manifests[metric_id]; if (metric == NULL) { return (enum metric_type)(-1); } @@ -1044,10 +690,7 @@ enum metric_type fieldstat_get_metric_type(const struct fieldstat *instance, int void fieldstat_get_cells_used_by_cube(const struct fieldstat *instance, int cube_id, struct fieldstat_tag_list **tag_list, size_t *n_cell) { - if (instance == NULL || cube_id < 0 || cube_id >= instance->max_n_cube) { - return; - } - const struct cube *cube = instance->cube[cube_id]; + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); if (cube == NULL) { return; } @@ -1060,10 +703,8 @@ int fieldstat_get_used_sampling(const struct fieldstat *instance, int cube_id) if (instance == NULL) { return FS_ERR_NULL_HANDLER; } - if (cube_id < 0 || cube_id >= instance->max_n_cube) { - return FS_ERR_INVALID_CUBE_ID; - } - const struct cube *cube = instance->cube[cube_id]; + + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); if (cube == NULL) { return FS_ERR_INVALID_CUBE_ID; } @@ -1076,12 +717,8 @@ int fieldstat_find_cube(const struct fieldstat *instance, const struct fieldstat if (instance == NULL) { return FS_ERR_NULL_HANDLER; } - const struct cube_manager *tag_cube_id_map = instance->shared_tag_cube_manager; - char *shared_tag_key = NULL; - size_t shared_tag_key_len = 0; - build_dynamic_cell_key(shared_tags, n_shared_tags, &shared_tag_key, &shared_tag_key_len); - int cube_id = cube_manager_find(tag_cube_id_map, shared_tag_key, shared_tag_key_len); - free(shared_tag_key); + + int cube_id = cube_manager_find(instance->cube_manager, shared_tags, n_shared_tags); if (cube_id == -1) { return FS_ERR_INVALID_KEY; } @@ -1091,5 +728,6 @@ int fieldstat_find_cube(const struct fieldstat *instance, const struct fieldstat void fieldstat_get_metric_in_cell(const struct fieldstat *instance, int cube_id, const struct fieldstat_tag_list *tags, int **metric_id_out, size_t *n_metric_out) { - return cube_get_cells_used_by_metric(instance->cube[cube_id], tags, metric_id_out, n_metric_out); + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + return cube_get_cells_used_by_metric(cube, tags, metric_id_out, n_metric_out); } \ No newline at end of file diff --git a/src/tags/heavy_keeper.c b/src/tags/heavy_keeper.c index 0769f4a..f887a94 100644 --- a/src/tags/heavy_keeper.c +++ b/src/tags/heavy_keeper.c @@ -585,12 +585,13 @@ unsigned my_max(unsigned a, unsigned b) { */ int heavy_keeper_add(struct heavy_keeper *heavy_keeper, const char *key, size_t key_len, int count, void *arg) { if (count == 0) { - const struct heap_entry *entry = sorted_set_find_entry(heavy_keeper->top_K_heap, key, key_len); - if (entry != NULL && !sorted_set_entry_dying(entry)) { - return 0; - } - if (sorted_set_cardinality(heavy_keeper->top_K_heap) < heavy_keeper->K) { + const struct heap_entry *entry = sorted_set_find_entry(heavy_keeper->top_K_heap, key, key_len); + + if (entry != NULL && !sorted_set_entry_dying(entry)) { + return 0; + } + sorted_set_insert(heavy_keeper->top_K_heap, key, key_len, count, arg); return 1; } @@ -610,13 +611,13 @@ int heavy_keeper_add(struct heavy_keeper *heavy_keeper, const char *key, size_t if (bucket->finger_print == FP) { // If a flow is not in the min-heap, then the estimated flow size should be no larger than nmin. // or if the min-heap is not full(nmin == 0), every flow should be taken into account, so of course it should be added. - // in neither case, bucket->count > nMin && not_in_sorted_set should happen. + // in neither case, bucket->count > nMin && not_in_sorted_set happen. // The flows whose counts are both larger than nmin and not in min-heap must have the same xxhash value, and its FP stored in bucket represents another tag, // In this case, the sketch won't be updated. This flow is expected to be taken into account in another array, // where its FP is different from the one it should collided with, so that element flows won't be missed. if (not_in_sorted_set) { - unsigned nMin = sorted_set_get_min_count(summary); - if (bucket->count > nMin) { + unsigned tmp_min = sorted_set_get_min_count(summary); + if (bucket->count > tmp_min) { continue; } } diff --git a/src/tags/tag_map.h b/src/tags/tag_map.h index 21c2716..2c8cd47 100644 --- a/src/tags/tag_map.h +++ b/src/tags/tag_map.h @@ -9,7 +9,7 @@ extern "C"{ #include "exdata.h" #include "my_ut_hash.h" -struct tag_map; +struct tag_map; // TODO: 重命名成hash table struct tag_map *tag_map_new(int max_query_num); void tag_map_set_exdata_schema(struct tag_map *pthis, exdata_new_cb new_fn, exdata_free_cb free_fn, exdata_merge_cb merge_fn, exdata_reset_cb reset_fn, exdata_copy_cb copy_fn); diff --git a/test/test_exporter_json.cpp b/test/test_exporter_json.cpp index 8369520..873abfe 100644 --- a/test/test_exporter_json.cpp +++ b/test/test_exporter_json.cpp @@ -813,14 +813,6 @@ TEST(export_test, enable_delta_and_reset_on_change_exporter_tag) { test_reset_one_round(trigger); } -TEST(export_test, enable_delta_and_reset_on_reset_instance) { - auto trigger = [](struct fieldstat *instance, struct fieldstat_json_exporter *fieldstat_json_exporter) { - fieldstat_reset(instance); - }; - - test_reset_one_round(trigger); -} - TEST(export_test, enable_delta_and_reset_on_delete_cube) { auto trigger = [](struct fieldstat *instance, struct fieldstat_json_exporter *fieldstat_json_exporter) { fieldstat_destroy_cube(instance, 0); diff --git a/test/test_register_and_reset.cpp b/test/test_register_and_reset.cpp index 8a32efa..11c8abb 100644 --- a/test/test_register_and_reset.cpp +++ b/test/test_register_and_reset.cpp @@ -4,43 +4,6 @@ #include "utils.hpp" -TEST(test_register, reset_and_version) -{ - struct fieldstat *instance = fieldstat_new(); - EXPECT_EQ(fieldstat_get_version(instance), 0); - fieldstat_reset(instance); - EXPECT_EQ(fieldstat_get_version(instance), 1); - fieldstat_reset(instance); - fieldstat_reset(instance); - EXPECT_EQ(fieldstat_get_version(instance), 3); - - fieldstat_free(instance); -} - -TEST(test_register, delete_cube_and_version_increase) -{ - struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - EXPECT_EQ(fieldstat_get_cube_version(instance, cube_id), 0); - int ret = fieldstat_destroy_cube(instance, cube_id); - EXPECT_EQ(ret, 0); - EXPECT_EQ(fieldstat_get_cube_version(instance, cube_id), FS_ERR_INVALID_CUBE_ID); - - cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - EXPECT_EQ(fieldstat_get_cube_version(instance, cube_id), 1); - - fieldstat_free(instance); -} - -TEST(test_register, query_on_wrong_version) -{ - EXPECT_EQ(fieldstat_get_cube_version(NULL, 1), FS_ERR_NULL_HANDLER); - struct fieldstat *instance = fieldstat_new(); - EXPECT_EQ(fieldstat_get_cube_version(instance, 1), FS_ERR_INVALID_CUBE_ID); - EXPECT_EQ(fieldstat_get_cube_version(instance, -1), FS_ERR_INVALID_CUBE_ID); - fieldstat_free(instance); -} - TEST(test_register, delete_cube_and_register_and_origin_position) { struct fieldstat *instance = fieldstat_new(); @@ -222,7 +185,7 @@ TEST(test_register, fork_registered_info_with_cube_and_metric) int metric_id2 = fieldstat_register_counter(instance, "counter2"); fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1); int cube_id_del = fieldstat_create_cube(instance, &TEST_TAG_DOUBLE_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int cube_id2 = fieldstat_create_cube(instance, &TEST_TAG_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + fieldstat_create_cube(instance, &TEST_TAG_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10); fieldstat_destroy_cube(instance, cube_id_del); struct fieldstat *dup = fieldstat_fork(instance); @@ -231,18 +194,23 @@ TEST(test_register, fork_registered_info_with_cube_and_metric) int cube_num; fieldstat_get_cubes(dup, &cube_ids, &cube_num); EXPECT_EQ(cube_num, 2); - EXPECT_EQ(cube_ids[0], cube_id); - EXPECT_EQ(cube_ids[1], cube_id2); + struct fieldstat_tag_list *tag_list = NULL; + + tag_list = fieldstat_get_shared_tags(dup, cube_ids[0]); + EXPECT_STREQ(tag_list->tag[0].key, TEST_SHARED_TAG.key); + fieldstat_tag_list_arr_free(tag_list, 1); + size_t n_cell = 0; + fieldstat_get_cells_used_by_cube(dup, cube_ids[0], &tag_list, &n_cell); + EXPECT_EQ(n_cell, 0); + + tag_list = fieldstat_get_shared_tags(dup, cube_ids[1]); + EXPECT_STREQ(tag_list->tag[0].key, TEST_TAG_DOUBLE.key); free(cube_ids); + fieldstat_tag_list_arr_free(tag_list, 1); EXPECT_STREQ(fieldstat_get_metric_name(dup, metric_id), "counter"); EXPECT_STREQ(fieldstat_get_metric_name(dup, metric_id2), "counter2"); - struct fieldstat_tag_list *tag_list = NULL; - size_t n_cell = 0; - fieldstat_get_cells_used_by_cube(dup, cube_id, &tag_list, &n_cell); - EXPECT_EQ(n_cell, 0); - fieldstat_free(dup); fieldstat_free(instance); } @@ -475,7 +443,6 @@ TEST(calibrate, master_change_cube) EXPECT_STREQ(tag_list->tag[0].key, TEST_TAG_STRING.key); EXPECT_EQ(fieldstat_find_cube(target, &TEST_TAG_STRING, 1), 0); - EXPECT_EQ(fieldstat_get_cube_version(target, cube_id), 1); fieldstat_free(master); fieldstat_free(target); @@ -551,18 +518,16 @@ TEST(calibrate, issue_calibrate_wrong) int n_cubes = 0; fieldstat_get_cubes(target, &cubes_id_ret, &n_cubes); EXPECT_EQ(n_cubes, 1); - EXPECT_EQ(cubes_id_ret[0], 0); - free(cubes_id_ret); - EXPECT_EQ(fieldstat_find_cube(target, tag_A, 1), FS_ERR_INVALID_KEY); - EXPECT_EQ(fieldstat_find_cube(target, tag_B, 1), 0); + EXPECT_EQ(fieldstat_find_cube(target, tag_B, 1), cubes_id_ret[0]); - struct fieldstat_tag_list *tag_list = fieldstat_get_shared_tags(target, 0); + struct fieldstat_tag_list *tag_list = fieldstat_get_shared_tags(target, cubes_id_ret[0]); EXPECT_STREQ(tag_list->tag[0].key, tag_B->key); fieldstat_tag_list_arr_free(tag_list, 1); fieldstat_free(master); fieldstat_free(target); + free(cubes_id_ret); } TEST(calibrate, delete_first_cube) -- cgit v1.2.3