diff options
| author | chenzizhan <[email protected]> | 2024-07-26 18:02:29 +0800 |
|---|---|---|
| committer | chenzizhan <[email protected]> | 2024-07-26 18:02:29 +0800 |
| commit | 240bbbb0e0409a6bca409eb319a0a00960cbc6fb (patch) | |
| tree | b15ddd24d6b851ccf797a5919df52d7994e4dbc9 | |
| parent | 3f46275f81d2d5af416f27fb24ab2c5ac21ec418 (diff) | |
| parent | 97e8724310c1a0d51600d723c6d3dcb6c4495d5f (diff) | |
Merge branch 'refactor-heavykeeper-newkey' into develop-version4
59 files changed, 6864 insertions, 10936 deletions
@@ -6,6 +6,7 @@ SI/ *.a *.d *.zip +*.pyc build/ .vscode .idea diff --git a/CMakeLists.txt b/CMakeLists.txt index 2513d1e..80c6cd3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_MACOSX_RPATH 0)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fPIC -Wall -lm -lz --std=gnu11")
-set(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} -Wall)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall")
# set(C_INCLUDE_PATH ${C_INCLUDE_PATH} /opt/MESA/include)
# set(CPLUS_INCLUDE_PATH ${CPLUS_INCLUDE_PATH} /opt/MESA/include)
@@ -98,14 +98,14 @@ include_directories(${PROJECT_SOURCE_DIR}/vendors) include_directories(${PROJECT_SOURCE_DIR}/src)
include_directories(${PROJECT_SOURCE_DIR}/include/fieldstat)
include_directories(${PROJECT_SOURCE_DIR}/src/metrics)
-include_directories(${PROJECT_SOURCE_DIR}/src/tags)
+include_directories(${PROJECT_SOURCE_DIR}/src/cells)
include_directories(${PROJECT_SOURCE_DIR}/src/utils)
file(GLOB SRC
"src/*.c"
"src/metrics/*.c"
- "src/tags/*.c"
+ "src/cells/*.c"
"src/exporter/*.c"
"src/utils/*.c"
"vendors/cjson/*.c"
diff --git a/ctest/CMakeLists.txt b/ctest/CMakeLists.txt index 26178b6..fd1077d 100644 --- a/ctest/CMakeLists.txt +++ b/ctest/CMakeLists.txt @@ -14,7 +14,6 @@ add_test(NAME COPY_GTEST_METRIC_COUNTER_BINARY COMMAND sh -c "cp ${CMAKE_BINARY_ add_test(NAME COPY_GTEST_METRIC_HISTOGRAM_BINARY COMMAND sh -c "cp ${CMAKE_BINARY_DIR}/test/test_metric_histogram ${CMAKE_BINARY_DIR}/testing/") add_test(NAME COPY_GTEST_METRIC_HLL_BINARY COMMAND sh -c "cp ${CMAKE_BINARY_DIR}/test/test_metric_hll ${CMAKE_BINARY_DIR}/testing/") add_test(NAME COPY_GTEST_REGISTER_AND_RESET_BINARY COMMAND sh -c "cp ${CMAKE_BINARY_DIR}/test/test_register_and_reset ${CMAKE_BINARY_DIR}/testing/") -add_test(NAME COPY_GTEST_UNIT_TEST_CELL_MANAGER_BINARY COMMAND sh -c "cp ${CMAKE_BINARY_DIR}/test/unit_test_cell_manager ${CMAKE_BINARY_DIR}/testing/") add_test(NAME COPY_GTEST_WRITE_JSON_FILE_BINARY COMMAND sh -c "cp ${CMAKE_BINARY_DIR}/test/test_write_json_file ${CMAKE_BINARY_DIR}/testing/") add_test(NAME CHMOD_UNITTEST COMMAND sh -c "chmod 0755 ${CMAKE_SOURCE_DIR}/test/test_fieldstat_exporter.py; cp ${CMAKE_SOURCE_DIR}/test/test_fieldstat_exporter.py ${CMAKE_BINARY_DIR}/testing/test_fieldstat_exporter") @@ -28,6 +27,5 @@ add_test(NAME GTEST_METRIC_COUNTER COMMAND test_metric_counter WORKING_DIRECTORY add_test(NAME GTEST_METRIC_HISTOGRAM COMMAND test_metric_histogram WORKING_DIRECTORY ${GTEST_RUN_DIR}) add_test(NAME GTEST_METRIC_HLL COMMAND test_metric_hll WORKING_DIRECTORY ${GTEST_RUN_DIR}) add_test(NAME GTEST_REGISTER_AND_RESET COMMAND test_register_and_reset WORKING_DIRECTORY ${GTEST_RUN_DIR}) -add_test(NAME GTEST_UNIT_TEST_CELL_MANAGER COMMAND unit_test_cell_manager WORKING_DIRECTORY ${GTEST_RUN_DIR}) add_test(NAME GTEST_WRITE_JSON_FILE COMMAND test_write_json_file WORKING_DIRECTORY ${GTEST_RUN_DIR}) add_test(NAME GTEST_PYTHON_EXPORTER COMMAND test_fieldstat_exporter WORKING_DIRECTORY ${GTEST_RUN_DIR}) diff --git a/include/fieldstat/fieldstat.h b/include/fieldstat/fieldstat.h index b0e679a..0bffe6c 100644 --- a/include/fieldstat/fieldstat.h +++ b/include/fieldstat/fieldstat.h @@ -13,9 +13,15 @@ extern "C" #define FS_ERR_NULL_HANDLER -2 #define FS_ERR_INVALID_CUBE_ID -3 #define FS_ERR_INVALID_METRIC_ID -4 -#define FS_ERR_INVALID_TAG -5 +#define FS_ERR_INVALID_DIMENSION -5 #define FS_ERR_INVALID_PARAM -6 -#define FS_ERR_INVALID_KEY -7 +#define FS_ERR_INVALID_METRIC_TYPE -8 +#define FS_ERR_MAX_N_CELL_LESS_THAN_ZERO -9 +#define FS_ERR_DIMENSION_ALREADY_EXISTS -10 +#define FS_ERR_METRIC_NAME_ALREADY_EXISTS -11 +#define FS_ERR_CUBE_SAMPLING_NOT_INITIALIZED -12 +#define FS_ERR_OPERATION_NOT_SUPPORTED_FOR_PRIMARY_METRIC -13 +#define FS_ERR_DIFFERENT_CONFIGURATION_FOR_SAME_CUBE -14 enum metric_type { @@ -24,21 +30,22 @@ enum metric_type METRIC_TYPE_HISTOGRAM, }; -enum fs_tag_type +enum field_type { - TAG_INTEGER, - TAG_DOUBLE, - TAG_CSTRING, + FIELD_VALUE_INTEGER, + FIELD_VALUE_DOUBLE, + FIELD_VALUE_CSTRING, }; enum sampling_mode { SAMPLING_MODE_COMPREHENSIVE, SAMPLING_MODE_TOPK, + SAMPLING_MODE_TOP_CARDINALITY, }; -struct fieldstat_tag { +struct field { const char *key; - enum fs_tag_type type; + enum field_type type; union{ long long value_longlong; double value_double; @@ -57,51 +64,37 @@ struct fieldstat *fieldstat_fork(const struct fieldstat *instance); * the configurations will be kept as much as possible, like cells in the same cube will be kept, but the cells in different cubes will be deleted. * @ return FS_OK or FS_ERR_NULL_HANDLER */ -int fieldstat_calibrate(const struct fieldstat *master, struct fieldstat *replica); +void fieldstat_calibrate(const struct fieldstat *master, struct fieldstat *replica); /* * @brief add an cube to this instance. Cube represents an template with a user-defined set of cells and metrics. - * @param shared_tags: tags that are shared by all cells in this cube. This is the key of the cube. Can be NULL. Must be unique. Shared_tags are ordered, which means that {"TAG_KEY": "123", "TAG_KEY2": "456"} and {"TAG_KEY2": "456", "TAG_KEY": "123"} are map to different cube. - * @param n_tag: number of shared tags. + * @param cube_dimensions: This is the key of the cube. Can be NULL. Must be unique. cube_dimensions are ordered, which means that {"KEY": "123", "KEY2": "456"} and {"KEY2": "456", "KEY": "123"} are map to different cube. + * @param n_dimension: number of field in dimension. * @param mode: sampling mode. Refer to enum sampling_mode. * @param max_n_cell: max number of samplings(cells) in each cube. When mode is TOPK, max_n_cell > 0, while in COMPREHENSIVE mode, max_n_cell can be 0, meaning that there is no limit. - * @return cube id, if success; otherwise, return FS_ERR_NULL_HANDLER, or FS_ERR_INVALID_PARAM when (max_n_cell == 0 && mode == TOPK). return FS_ERR_INVALID_KEY when the shared_tags is not unique. + * @return cube id, if success; otherwise, return FS_ERR_NULL_HANDLER, or FS_ERR_INVALID_PARAM when (max_n_cell == 0 && mode == TOPK). return FS_ERR_INVALID_KEY when the cube_dimensions is not unique. */ -int fieldstat_create_cube(struct fieldstat *instance, const struct fieldstat_tag *shared_tags, size_t n_tag, enum sampling_mode mode, size_t max_n_cell); - -/* - @brief Change the topk cube primary metric id. When fieldstat_counter_add or fieldstat_counter_set are called on the primary metric, the topk record of such cell will be updated. - the default primary metric id is 0. - @return FS_OK, FS_ERR_NULL_HANDLER or FS_ERR_INVALID_CUBE_ID. - FS_ERR_INVALID_METRIC_ID when the metric is not registered to instance. - FS_ERR_INVALID_PARAM when the cube is not a topk sampling cube, or the metric is not a counter. - -*/ -int fieldstat_cube_set_primary_metric(struct fieldstat *instance, int cube_id, int metric_id); +int fieldstat_cube_create(struct fieldstat *instance, const struct field *cube_dimensions, size_t n_dimension); +int fieldstat_cube_set_sampling(struct fieldstat *instance, int cube_id, enum sampling_mode mode, int max_n_cell, int primary_metric_id); /* * @brief Delete the cube of cube_id. All the cells and metrics are deleted. The cube_id may be reused by other new cubes. Increase the corresponding cube_version by 1. * @return FS_OK, FS_ERR_NULL_HANDLER or FS_ERR_INVALID_CUBE_ID */ -int fieldstat_destroy_cube(struct fieldstat *instance, int cube_id); +int fieldstat_cube_destroy(struct fieldstat *instance, int cube_id); /* - * @brief get the cube_version of the cube of cube_id. - * @return cube_version if success. FS_ERR_NULL_HANDLER or FS_ERR_INVALID_CUBE_ID if fail. -*/ -long long fieldstat_get_cube_version(const struct fieldstat *instance, int cube_id); -/* * @brief add a metric to the cube of cube_id. One metric may be associated with different cells. * @param metric_name: name of the metric. Cannot be NULL. Must be unique. * @return metric id>=0 if success. If failed, return FS_ERR_NULL_HANDLER, FS_ERR_INVALID_KEY(when metric_name is not unique in this cube) */ -int fieldstat_register_counter(struct fieldstat *instance, const char *metric_name); +int fieldstat_register_counter(struct fieldstat *instance, int cube_id, const char *metric_name); /* * @brief refer to fieldstat_register_counter. * @param precision: the bigger, the larger memory consumption, while accuracy improved. Must be in [4, 18]. * @return metric id if success. If failed, return FS_ERR_NULL_HANDLER, FS_ERR_INVALID_KEY(when metric_name is not unique in this cube), or FS_ERR_INVALID_PARAM(if precision not in range) */ -int fieldstat_register_hll(struct fieldstat *instance, const char *metric_name, unsigned char precision); +int fieldstat_register_hll(struct fieldstat *instance, int cube_id, const char *metric_name, unsigned char precision); /* * @brief refer to fieldstat_register_counter. @@ -110,11 +103,11 @@ int fieldstat_register_hll(struct fieldstat *instance, const char *metric_name, * @param significant_figures: the precision of the histogram. Must be in [1, 5]. * @return metric id if success. If failed, return FS_ERR_NULL_HANDLER, FS_ERR_INVALID_KEY(when metric_name is not unique in this cube), or FS_ERR_INVALID_PARAM(if any of the 3 params are out of range) */ -int fieldstat_register_hist(struct fieldstat *instance, const char *metric_name, long long lowest_trackable_value, long long highest_trackable_value, int significant_figures); +int fieldstat_register_histogram(struct fieldstat *instance, int cube_id, const char *metric_name, long long lowest_trackable_value, long long highest_trackable_value, int significant_figures); /* * @brief let the value of counter metric of cell_id increase by `increment`. - * @param cube_id: cube id, previously returned by fieldstat_create_cube. + * @param cube_id: cube id, previously returned by fieldstat_cube_create. * @param metric_id: metric id, previously returned by fieldstat_register_counter. * @param increment: increment of the counter metric. Can be negative. * @return FS_OK if success. FS_ERR_NULL_HANDLER, FS_ERR_INVALID_CUBE_ID, FS_ERR_INVALID_METRIC_ID if fail. @@ -123,14 +116,14 @@ int fieldstat_register_hist(struct fieldstat *instance, const char *metric_name, * In comprehensive mode, a full cube cannot be added any more cells. * In topk mode, every increment matters, so even the cube is full, the cell can also replace the least frequent cell. */ -int fieldstat_counter_incrby(struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag *tags, size_t n_tag, long long increment); +int fieldstat_counter_incrby(struct fieldstat *instance, int cube_id, int metric_id, const struct field *cell_dimensions, size_t n_dimensions, long long increment); /* * @brief let the value of counter metric equal to value. Other annotations refer to fieldstat_counter_incrby. * @return Refer to fieldstat_counter_incrby. What's more, be cautious to call this function if the metric is a primary metric of a topk cube, * in such case, FS_ERR_INVALID_PARAM will be the output when the value is set to a smaller one(increment is negative). */ -int fieldstat_counter_set(struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag *tags, size_t n_tag, long long value); +int fieldstat_counter_set(struct fieldstat *instance, int cube_id, int metric_id, const struct field *cell_dimensions, size_t n_dimensions, long long value); /* * @brief add a key to the hll metric of cell_id. HLL approximates the number of distinct elements in a set of `key`s. @@ -139,7 +132,9 @@ int fieldstat_counter_set(struct fieldstat *instance, int cube_id, int metric_id * @return FS_OK if success. FS_ERR_NULL_HANDLER, FS_ERR_INVALID_CUBE_ID, FS_ERR_INVALID_METRIC_ID if fail. * Since topk only support counter, FS_ERR_INVALID_PARAM is returned when the cube is topk. */ -int fieldstat_hll_add(struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag *tags, size_t n_tag, const char *key, size_t key_len); +int fieldstat_hll_add(struct fieldstat *instance, int cube_id, int metric_id, const struct field *cell_dimensions, size_t n_dimensions, const char *key, size_t key_len); +int fieldstat_hll_add_field(struct fieldstat *instance, int cube_id, int metric_id, const struct field *cell_dimensions, size_t n_dimensions, const struct field *item, size_t item_len); + /* * @brief Add a value to the histogram metric of cell_id. Histogram will record the distribution of the values. @@ -148,32 +143,28 @@ int fieldstat_hll_add(struct fieldstat *instance, int cube_id, int metric_id, co * @return FS_OK if success. FS_ERR_NULL_HANDLER, FS_ERR_INVALID_CUBE_ID, FS_ERR_INVALID_METRIC_ID if fail. * FS_ERR_INVALID_PARAM when value is less than 0, or the cube is topk. */ -int fieldstat_hist_record(struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag *tags, size_t n_tag, long long value); +int fieldstat_histogram_record(struct fieldstat *instance, int cube_id, int metric_id, const struct field *cell_dimensions, size_t n_dimensions, long long value); /* * @brief Delete all the cells, also the content of every metrics. The cube and metrics are not deleted. Increase cell_version by 1. Note that the cell record won't be deleted at once, they just seem to be deleted. The cell record will be deleted when they are not used since the last reset and until the next reset. */ void fieldstat_reset(struct fieldstat *instance); -/* - version is increased by 1 when we call fieldstat_reset. -*/ -unsigned long fieldstat_get_version(const struct fieldstat *instance); /* @brief Merge the instance. The registered cubes and metrics are merged even if there are no cells added. @return 0 if success. return FS_ERR_INVALID_PARAM when the registered cubes or metrics between dest and src of the same keys has different types or configurations(like different primary metric). */ -int fieldstat_merge(struct fieldstat *instance, struct fieldstat *src); +int fieldstat_merge(struct fieldstat *instance, const struct fieldstat *src); /* -------------------------------------------------------------------------- */ /* query */ /* -------------------------------------------------------------------------- */ -struct fieldstat_tag_list +struct field_list { - struct fieldstat_tag *tag; - size_t n_tag; + struct field *field; + size_t n_field; }; /* @@ -184,50 +175,34 @@ struct fieldstat_tag_list void fieldstat_get_cubes(const struct fieldstat *instance, int **cube_ids, int *n_cube); /* - * @brief Get all the registered metrics of a cube. - * @return return FS_ERR_NULL_HANDLER or FS_ERR_INVALID_CUBE_ID. Or FS_OK + * @brief Get all the registered metrics by fieldstat_register_counter, fieldstat_register_hll, fieldstat_register_histogram. */ -int fieldstat_get_metrics_used_by_cube(const struct fieldstat *instance, int cube_id, int **metric_id_out, size_t *n_metric); +void fieldstat_cube_get_metrics(const struct fieldstat *instance, int cube_id, int **metric_id_out, size_t *n_metric); -/* - * @brief Get all the registered metrics by fieldstat_register_counter, fieldstat_register_hll, fieldstat_register_hist. -*/ -void fieldstat_get_metrics(const struct fieldstat *instance, int **metric_id_out, size_t *n_metric); +void fieldstat_get_metric_in_cell(const struct fieldstat *instance, int cube_id, const struct field_list *cell_dimensions, int **metric_id_out, size_t *n_metric_out); // query the name of the metric, return NULL if metric_id is invalid. -const char *fieldstat_get_metric_name(const struct fieldstat *instance, int metric_id); +const char *fieldstat_get_metric_name(const struct fieldstat *instance, int cube_id, int metric_id); // query the type of the metric. return (enum metric_type)-1 if metric_id is invalid. -enum metric_type fieldstat_get_metric_type(const struct fieldstat *instance, int metric_id); - -/* - get the tags added to metric when calling fieldstat_counter_incrby, fieldstat_counter_set, fieldstat_hll_add, fieldstat_hist_record. -*/ -void fieldstat_get_cells_used_by_metric(const struct fieldstat *instance, int cube_id, int metric_id, - struct fieldstat_tag_list **tag_list, size_t *n_cell); +enum metric_type fieldstat_get_metric_type(const struct fieldstat *instance, int cube_id, int metric_id); /* - get the tags added to cube when calling fieldstat_counter_incrby, fieldstat_counter_set, fieldstat_hll_add, fieldstat_hist_record. + get the cell_dimensions added to cube when calling fieldstat_counter_incrby, fieldstat_counter_set, fieldstat_hll_add, fieldstat_histogram_record. */ -void fieldstat_get_cells_used_by_cube(const struct fieldstat *instance, int cube_id, struct fieldstat_tag_list **tag_list, size_t *n_cell); +void fieldstat_cube_get_cells(const struct fieldstat *instance, int cube_id, struct field_list **cell_dimensions, size_t *n_cell); /* - get the shared tag of fieldstat_create_cube. User free them by calling fieldstat_tag_list_arr_free(struct fieldstat_tag_list *, 1) + get the field of fieldstat_cube_create. User free them by calling fieldstat_field_list_arr_free(struct field_list *, 1) return NULL when ID is invalid. */ -struct fieldstat_tag_list *fieldstat_get_shared_tags(const struct fieldstat *instance, int cube_id); +struct field_list *fieldstat_cube_get_dimensions(const struct fieldstat *instance, int cube_id); -/* - return a cube id corresponding to the shared tags. FS_ERR_INVALID_KEY is returned if the shared tags are not found. -*/ -int fieldstat_find_cube(const struct fieldstat *instance, const struct fieldstat_tag *shared_tags, size_t n_shared_tags); /* - return FS_ERR_NULL_HANDLER, FS_ERR_INVALID_CUBE_ID if fail. - When return OK, mode == COMPREHENSIVE, primary_metric_id == -1. - When return OK, mode == TOPK, primary_metric_id >= 0. + return a cube id corresponding to `cube_dimensions`. FS_ERR_INVALID_KEY is returned if the cube is not found. */ -int fieldstat_get_cube_mode(const struct fieldstat *instance, int cube_id, enum sampling_mode *mode, int *primary_metric_id); +int fieldstat_find_cube(const struct fieldstat *instance, const struct field *cube_dimensions, size_t n_dimensions); /* get the cell numbers in a cube. Return FS_ERR_INVALID_CUBE_ID if cube_id is invalid. @@ -238,24 +213,24 @@ int fieldstat_get_used_sampling(const struct fieldstat *instance, int cube_id); * @brief Get the value of a metric of a cell. * @param cube_id: cube id, previously returned by fieldstat_get_cubes. * @param metric_id: metric id, previously returned by fieldstat_get_max_metric_id. - * @param tags: previously returned by fieldstat_get_cells_used_by_metric. + * @param cell_dimensions: previously returned by fieldstat_get_cells_used_by_metric. * @param value_out: the value of the metric. If the cell is not found, *value_out is set to 0. * @return FS_OK if success. FS_ERR_NULL_HANDLER, FS_ERR_INVALID_CUBE_ID, FS_ERR_INVALID_METRIC_ID if fail. */ -int fieldstat_counter_get(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tags, long long *value); +int fieldstat_counter_get(const struct fieldstat *instance, int cube_id, const struct field_list *cell_dimensions, int metric_id, long long *value); /* @brief Get an approximate count of the number of distinct elements in the cell. Other information refer to fieldstat_counter_get. @return >= 0 if success. FS_ERR_INVALID_PARAM if precision is invalid. */ -int fieldstat_hll_get(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tags, double *value); -long long fieldstat_hist_value_at_percentile(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tags, double percentile); -long long fieldstat_hist_count_le_value(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tags, long long value); +int fieldstat_hll_get(const struct fieldstat *instance, int cube_id, const struct field_list *cell_dimensions, int metric_id, double *value); +long long fieldstat_histogram_value_at_percentile(const struct fieldstat *instance, int cube_id, const struct field_list *cell_dimensions, int metric_id, double percentile); +long long fieldstat_histogram_count_le_value(const struct fieldstat *instance, int cube_id, const struct field_list *cell_dimensions, int metric_id, long long value); // get the base 64 encoded string of the serialized blob of a cell -void fieldstat_get_serialized_blob(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tags, char **blob, size_t *blob_size); +void fieldstat_metric_get_serialization_as_base64(const struct fieldstat *instance, int cube_id, int metric_id, const struct field_list *cell_dimensions, char **blob, size_t *blob_size); -void fieldstat_tag_list_arr_free(struct fieldstat_tag_list *tag_list, size_t n_cell); +void fieldstat_field_list_arr_free(struct field_list *field_lists, size_t n_field_list); #ifdef __cplusplus diff --git a/include/fieldstat/fieldstat_easy.h b/include/fieldstat/fieldstat_easy.h index 7403c3f..97d7394 100644 --- a/include/fieldstat/fieldstat_easy.h +++ b/include/fieldstat/fieldstat_easy.h @@ -13,9 +13,9 @@ struct fieldstat_easy; * new a fieldstat_easy instance. * @param max_thread_num: max thread number of this instance. * @param name: name of this instance. Will appear in output json. Can be NULL. - * @param tags: tags of this instance. Will appear in output json. Can be NULL. + * @param dimensions: dimensions of this instance. Will appear in output json. Can be NULL. */ -struct fieldstat_easy *fieldstat_easy_new(int max_thread_num, const char *name, const struct fieldstat_tag *tags, size_t n_tag); +struct fieldstat_easy *fieldstat_easy_new(int max_thread_num, const char *name, const struct field *global_dimensions, size_t n_dimension); /* * free a fieldstat_easy instance. */ @@ -29,16 +29,6 @@ void fieldstat_easy_free(struct fieldstat_easy *fse); int fieldstat_easy_enable_auto_output(struct fieldstat_easy *pthis, const char *output_path, int interval_second); /* - affect the output of fieldstat_easy_output and fieldstat_easy_output_array. - let json exporter output delta value by the side of the original value(accumulated). If a cell / metric is new, the delta value is the same as the original value. - Outputting delta value is disabled by default. - When the fieldstat instance is reset, the delta value will be reset. (next output will be the original value) - Only affects the metrics of counter type. - Outputting delta value is time-consuming. Be cautious when enabling it. -*/ -void fieldstat_easy_enable_delta_in_active_output(struct fieldstat_easy *fse); - -/* * @brief add a metric to the cube of cube_id. One metric may be associated with different cells. * @param metric_name: name of the metric. Cannot be NULL. Must be unique. * @return metric id>=0 if success. If failed, return FS_ERR_NULL_HANDLER, FS_ERR_INVALID_KEY(when metric_name is not unique in this cube). @@ -67,19 +57,25 @@ void fieldstat_easy_output_array(struct fieldstat_easy *fse, char ***json_object */ int fieldstat_easy_output_array_and_reset(struct fieldstat_easy *fse, char ***json_objects, size_t *n_object); /* - * @brief let the value of counter metric of tags increase by `increment`. + * @brief let the value of counter metric of dimensions increase by `increment`. * @param thread_id: thread id. Must be in [0, max_thread_num). * @param metric_id: metric id, previously returned by fieldstat_easy_register_counter. * @param increment: increment of the counter metric. Can be negative. * return -1 also when the thread_id is out of range.FS_ERR_INVALID_METRIC_ID metric_id is not registered. */ -int fieldstat_easy_counter_incrby(struct fieldstat_easy *fse, int thread_id, int metric_id, const struct fieldstat_tag *tags, size_t n_tag, long long increment); + +int fieldstat_easy_counter_incrby(struct fieldstat_easy *fse, int thread_id, int metric_id, const struct field *dimensions, size_t n_dimensions, long long increment); + +// TODO: easy 的加速使用，跟其他人对接一下。包括如何在不改变fieldstat.h 的基础上再来一套接口。 +// int fieldstat_easy_counter_incrby(struct fieldstat_easy *fse, int thread_id, int metric_id, int cell_id, long long increment); +// int fieldstat_easy_add_cell(struct fieldstat_easy *fse,const struct field *dimensions, size_t n_dimensions); + /* - * @brief let the value of counter metric of tags equal to `value`. + * @brief let the value of counter metric of dimensions equal to `value`. * for other notes, see fieldstat_easy_counter_incrby. * The value will be output by summing each ones in different threads, exactly the same as values set by fieldstat_easy_counter_incrby. */ -int fieldstat_easy_counter_set(struct fieldstat_easy *fse, int thread_id, int metric_id, const struct fieldstat_tag *tags, size_t n_tag, long long value); +int fieldstat_easy_counter_set(struct fieldstat_easy *fse, int thread_id, int metric_id, const struct field *dimensions, size_t n_dimensions, long long value); /* * @brief Add a value to the histogram metric of cell_id. Histogram will record the distribution of the values. The value bigger than highest_trackable_value will be set to highest_trackable_value. The value less than lowest_trackable_value will be tried to record, and, if succeed, remains in the record as -inf(most of the time) or 0(if value == 0) @@ -87,7 +83,7 @@ int fieldstat_easy_counter_set(struct fieldstat_easy *fse, int thread_id, int me * @return FS_OK if success. FS_ERR_NULL_HANDLER, FS_ERR_INVALID_CUBE_ID, FS_ERR_INVALID_METRIC_ID if fail. * return -1 also when the thread_id is out of range. */ -int fieldstat_easy_histogram_record(struct fieldstat_easy *fse, int thread_id, int metric_id, const struct fieldstat_tag *tags, size_t n_tag, long long value); +int fieldstat_easy_histogram_record(struct fieldstat_easy *fse, int thread_id, int metric_id, const struct field *dimensions, size_t n_dimensions, long long value); #ifdef __cplusplus } diff --git a/include/fieldstat/fieldstat_exporter.h b/include/fieldstat/fieldstat_exporter.h index f7db70a..2df08ab 100644 --- a/include/fieldstat/fieldstat_exporter.h +++ b/include/fieldstat/fieldstat_exporter.h @@ -14,20 +14,64 @@ extern "C" struct fieldstat_json_exporter; struct fieldstat_json_exporter *fieldstat_json_exporter_new(); -void fieldstat_json_exporter_set_global_tag(struct fieldstat_json_exporter *exporter, const struct fieldstat_tag tag_list[], size_t n_tag); +void fieldstat_json_exporter_set_global_tag(struct fieldstat_json_exporter *exporter, const struct field tag_list[], size_t n_field); void fieldstat_json_exporter_set_name(struct fieldstat_json_exporter *exporter, const char *name); void fieldstat_json_exporter_free(struct fieldstat_json_exporter *exporter); /* Output the fieldstat instance to json string array. User must free the output string. + format: + [ + { + "name":"exporter_name", + "tags":[ + {"device ID":123}, + {"CLIENT IP":"123.1.1.1"} + ], + "fields":[ + {"in_bytes":1024}, + {"in_pkts":123}, + {"sessions":<hll blob base64-encoded string>} + ] + "timestamp_ms":123456789 + }. + { + "name":"exporter_name", + "tags":[ + {"device ID":123}, + {"CLIENT IP":"1.2.3.4"} + ], + "fields":[ + {"in_bytes":1}, + {"in_pkts":2}, + ] + "timestamp_ms":123456789 + } + ] */ char *fieldstat_json_exporter_export(const struct fieldstat_json_exporter *exporter, const struct fieldstat *instance, const struct timeval *timestamp); /* - after call fieldstat_json_exporter_export_array, user must free the output array and each string. + after call fieldstat_json_exporter_export_array, user must free the output array and each string. output[0] = + "{"name":"exporter_name", "tags":[{"device ID":123}, {"CLIENT IP":"1.2.3.4"}],"fields":[{"in_bytes":1}, {"in_pkts":2},]"timestamp_ms":123456789}" + The same as every object in the array output by `fieldstat_json_exporter_export`. */ void fieldstat_json_exporter_export_array(const struct fieldstat_json_exporter *exporter, const struct fieldstat *instance, const struct timeval *timestamp, char ***output, size_t *output_size); /* +the content is the same, but no nested format. flat format : +{ +"name":"exporter_name", +"device ID":123, +"in_bytes":1, +"timestamp_ms":123456789 +} +do not support fieldstat_json_exporter_enable_delta. + +if a cell has no metric record, the cell will not be exported. (The same as fieldstat_json_exporter_export_array) +*/ +void fieldstat_json_exporter_export_flat_array(const struct fieldstat_json_exporter *exporter, const struct fieldstat *instance, const struct timeval *timestamp, char ***output, size_t *output_size); +char *fieldstat_json_exporter_export_flat(const struct fieldstat_json_exporter *exporter, const struct fieldstat *instance, const struct timeval *timestamp); +/* let json exporter output delta value by the side of the original value(accumulated). If a cell / metric is new, the delta value is the same as the original value. Outputting delta value is disabled by default. When the exporter name or exporter global tags are changed, or the fieldstat instance is reset, or one of the cube is deleted, the delta value will be reset. (next output will be the original value) diff --git a/my_ut_hash.txt b/my_ut_hash.txt deleted file mode 100644 index 115a4a8..0000000 --- a/my_ut_hash.txt +++ /dev/null @@ -1,4424 +0,0 @@ - -my_ut_hash.c.o: file format elf64-x86-64 - -Contents of section .text: - 0000 8b4710c3 66662e0f 1f840000 00000090 .G..ff.......... - 0010 41564155 41545553 4c8b7708 488b4608 AVAUATUSL.w.H.F. - 0020 4d85f675 1b4531ed 4885c041 0f95c55b M..u.E1.H..A...[ - 0030 4489e85d 415c415d 415ec30f 1f440000 D..]A\A]A^...D.. - 0040 4939c675 6a488b2e 488b1f45 31e4eb31 I9.ujH..H..E1..1 - 0050 83f80275 1b488b75 10488b7b 10e80000 ...u.H.u.H.{.... - 0060 000085c0 7549662e 0f1f8400 00000000 ....uIf......... - 0070 4983c401 4883c518 4883c318 4d39e674 I...H...H...M9.t - 0080 ae488b75 00488b3b e8000000 004189c5 .H.u.H.;.....A.. - 0090 85c0751b 8b43083b 45087513 83f80174 ..u..C.;E.u....t - 00a0 2785c075 ab488b45 10483943 1074c141 '..u.H.E.H9C.t.A - 00b0 bd010000 005b5d44 89e8415c 415d415e .....[]D..A\A]A^ - 00c0 c30f1f80 00000000 f20f1043 10660f2e ...........C.f.. - 00d0 45107adb 749aebd7 0f1f8400 00000000 E.z.t........... - 00e0 554889fd 534889f3 4883ec08 488b3ee8 UH..SH..H...H.>. - 00f0 00000000 48894500 8b430889 45088b43 ....H.E..C..E..C - 0100 0883f801 742285c0 743683f8 02750d48 ....t"..t6...u.H - 0110 8b7b10e8 00000000 48894510 4883c408 .{......H.E.H... - 0120 5b5dc30f 1f440000 f20f1043 10f20f11 []...D.....C.... - 0130 45104883 c4085b5d c30f1f80 00000000 E.H...[]........ - 0140 488b4310 48894510 4883c408 5b5dc390 H.C.H.E.H...[].. - 0150 4885ff74 5b415449 89fc5553 85f67e41 H..t[ATI..US..~A - 0160 8d46ff48 89fb488d 0440488d 6cc718eb [email protected]... - 0170 100f1f80 00000000 4883c318 4839eb74 ........H...H9.t - 0180 20488b3b e8000000 00837b08 0275e948 H.;......{..u.H - 0190 8b7b1048 83c318e8 00000000 4839eb75 .{.H........H9.u - 01a0 e05b4c89 e75d415c e9000000 000f1f00 .[L..]A\........ - 01b0 c366662e 0f1f8400 00000000 0f1f4000 .ff...........@. - 01c0 554889f5 534889fb 4883ec08 488b3648 UH..SH..H...H.6H - 01d0 8b3fe800 00000085 c075188b 53088b4d .?.......u..S..M - 01e0 0839ca75 1b83fa01 744e85d2 743a83fa .9.u....tN..t:.. - 01f0 02741d48 83c4085b 5dc3660f 1f440000 .t.H...[].f..D.. - 0200 4883c408 89d029c8 5b5dc30f 1f440000 H.....).[]...D.. - 0210 488b7510 488b7b10 4883c408 5b5de900 H.u.H.{.H...[].. - 0220 0000000f 1f440000 488b4310 2b4510eb .....D..H.C.+E.. - 0230 c20f1f80 00000000 f20f1043 10f20f5c ...........C...\ - 0240 4510f20f 2cc0ebab 0f1f8400 00000000 E...,........... - 0250 89f0c1e0 0429f02d 3b62e37e 8907c390 .....).-;b.~.... - 0260 8b074885 d2741d48 01f2660f 1f440000 ..H..t.H..f..D.. - 0270 0fbe0e48 83c60131 c869c093 01000148 ...H...1.i.....H - 0280 39f275ec 8907c366 0f1f8400 00000000 9.u....f........ - 0290 8b07c366 662e0f1f 84000000 00006690 ...ff.........f. - 02a0 41554154 554889f5 89d65348 89fb4883 AUATUH....SH..H. - 02b0 ec184c8d 64240c4c 89e7e800 00000048 ..L.d$.L.......H - 02c0 85ed7476 4883c310 488d446d 004c8d2c ..tvH...H.Dm.L., - 02d0 c3eb1e0f 1f440000 ba080000 004889de .....D.......H.. - 02e0 4c89e7e8 00000000 4883c318 4939dd74 L.......H...I9.t - 02f0 49488b6b f04889ef e8000000 004889ee IH.k.H.......H.. - 0300 4c89e748 89c2e800 0000008b 43f883f8 L..H........C... - 0310 0176c583 f80275d0 488b2b48 83c31848 .v....u.H.+H...H - 0320 89efe800 00000048 89ee4c89 e74889c2 .......H..L..H.. - 0330 e8000000 004939dd 75b74c89 e7e80000 .....I9.u.L..... - 0340 00004883 c4185b5d 415c415d c30f1f00 ..H...[]A\A].... - 0350 4157660f efc08d82 28442324 41564155 AWf.....(D#$AVAU - 0360 41545553 4881ec88 0000000f 29442450 ATUSH.......)D$P - 0370 0f294424 60894424 588d8277 caeb8589 .)D$`.D$X..w.... - 0380 54246081 c24f86c8 610f2944 24708b4c T$`..O..a.)D$p.L - 0390 24788944 245c660f 6f642450 89542464 $x.D$\f.od$P.T$d - 03a0 660f6f6c 24604189 cd0f2964 24200f29 f.ol$`A...)d$ .) - 03b0 6c243048 c7442440 00000000 894c2448 [email protected]$H - 03c0 4885f60f 84f70900 00488d5f 08488d04 H........H._.H.. - 03d0 76448b64 24208b6c 2424488d 04c34889 vD.d$ .l$$H...H. - 03e0 44241048 8d442420 48894424 080f1f00 D$.H.D$ H.D$.... - 03f0 4c8b7bf8 4c89ffe8 00000000 458d0c04 L.{.L.......E... - 0400 4883f80f 4989c60f 97c04183 f90f4489 H...I.....A...D. - 0410 4c24200f 97c209d0 0fb6c009 e8894424 L$ ...........D$ - 0420 244489e8 498d1406 4883fa0f 0f866603 $D..I...H.....f. - 0430 00004c89 fe4585ed 0f850206 00004b8d ..L..E........K. - 0440 04374c8d 48f04939 f10f82b9 00000044 .7L.H.I9.......D - 0450 8b442428 448b5424 2c4889f1 8b7c2430 .D$(D.T$,H...|$0 - 0460 448b5c24 340f1f00 691177ca eb854883 D.\$4...i.w...H. - 0470 c1104101 d06951f4 77caeb85 41c1c00d ..A..iQ.w...A... - 0480 4569c0b1 79379e41 01d26951 f877caeb Ei..y7.A..iQ.w.. - 0490 8541c1c2 0d4569d2 b179379e 01d76951 .A...Ei..y7...iQ - 04a0 fc77caeb 85c1c70d 69ffb179 379e4401 .w......i..y7.D. - 04b0 dac1c20d 4469dab1 79379e49 39c973a8 ....Di..y7.I9.s. - 04c0 897c2418 660f6e4c 24184929 f144895c .|$.f.nL$.I).D.\ - 04d0 2418660f 6e542418 4983e1f0 44894424 $.f.nT$.I...D.D$ - 04e0 18660f6e 4424184a 8d740e10 44895424 .f.nD$.J.t..D.T$ - 04f0 18660f6e 5c241866 0f62ca66 0f62c366 .f.n\$.f.b.f.b.f - 0500 0f6cc10f 11442428 4531ed48 39f00f87 .l...D$(E1.H9... - 0510 cc050000 8b0383f8 010f84a6 02000085 ................ - 0520 c00f84f9 03000083 f8020f85 e0030000 ................ - 0530 4c8b7308 4c89f7e8 00000000 448b6424 L.s.L.......D.d$ - 0540 204989c2 4d8d3c06 4101c448 83f80f0f I..M.<.A..H.... - 0550 97c04183 fc0f4489 64242041 0f97c044 ..A...D.d$ A...D - 0560 09c00fb6 c00b4424 2489c589 44242444 ......D$$...D$$D - 0570 89e8498d 14024883 fa0f0f86 00080000 ..I...H......... - 0580 4585ed0f 853f0700 004d8d57 f04d39f2 E....?...M.W.M9. - 0590 0f82b200 00008b54 24288b7c 242c4c89 .......T$(.|$,L. - 05a0 f08b7424 30448b5c 2434660f 1f440000 ..t$0D.\$4f..D.. - 05b0 690877ca eb854883 c01001ca 6948f477 i.w...H.....iH.w - 05c0 caeb85c1 c20d69d2 b179379e 01cf6948 ......i..y7...iH - 05d0 f877caeb 85c1c70d 69ffb179 379e01ce .w......i..y7... - 05e0 6948fc77 caeb85c1 c60d69f6 b179379e iH.w......i..y7. - 05f0 4401d9c1 c10d4469 d9b17937 9e4939c2 D.....Di..y7.I9. - 0600 73ae8974 2418660f 6e4c2418 4d29f244 s..t$.f.nL$.M).D - 0610 895c2418 660f6e64 24184983 e2f08954 .\$.f.nd$.I....T - 0620 2418660f 6e442418 4f8d7416 10897c24 $.f.nD$.O.t...|$ - 0630 18660f6e 6c241866 0f62cc66 0f62c566 .f.nl$.f.b.f.b.f - 0640 0f6cc10f 11442428 4531ed4d 39f70f87 .l...D$(E1.M9... - 0650 4c060000 0f1f4000 4883c318 483b5c24 [email protected];\$ - 0660 100f8589 fdffff8b 7424308d 86b16756 ........t$0...gV - 0670 1685ed0f 858f0400 00468d0c 20488b5c .........F.. H.\ - 0680 24084489 e883e00f 4183e50c 488d7318 $.D.....A...H.s. - 0690 746e6954 24383dae b2c2488d 48fc4101 tniT$8=...H.H.A. - 06a0 d141c1c9 0f4569c9 2febd427 4883f903 .A...Ei./..'H... - 06b0 76366954 243c3dae b2c24883 e8084101 v6iT$<=...H...A. - 06c0 d141c1c9 0f4569c9 2febd427 4883f803 .A...Ei./..'H... - 06d0 76166944 24403dae b2c24101 c141c1c9 v.iD$@=...A..A.. - 06e0 0f4569c9 2febd427 4889c848 8b5c2408 .Ei./..'H..H.\$. - 06f0 48c1e802 488d7483 1c48f7d8 488d0481 H...H.t..H..H... - 0700 4885c074 530fb616 69d2b167 56164101 H..tS...i..gV.A. - 0710 d141c1c1 0b4569c9 b179379e 4883e801 .A...Ei..y7.H... - 0720 74360fb6 560169d2 b1675616 4101d141 t6..V.i..gV.A..A - 0730 c1c10b45 69c9b179 379e4883 f8017418 ...Ei..y7.H...t. - 0740 0fb64602 69c0b167 56164101 c141c1c1 ..F.i..gV.A..A.. - 0750 0b4569c9 b179379e 4489c848 81c48800 .Ei..y7.D..H.... - 0760 0000c1e8 0f5b5d41 31c1415c 415d4169 .....[]A1.A\A]Ai - 0770 c177caeb 85415e41 5f4189c1 41c1e90d .w...A^A_A..A... - 0780 4431c869 c03daeb2 c289c2c1 ea1031d0 D1.i.=........1. - 0790 c30f1f80 00000000 488b7424 084c89f2 ........H.t$.L.. - 07a0 488d7c06 184c89fe e8000000 00448b6c H.|..L.......D.l - 07b0 24488b03 4501f544 896c2448 83f8010f $H..E..D.l$H.... - 07c0 855afdff ff8b4424 20448d60 0831c041 .Z....D$ D.`.1.A - 07d0 83fc0f44 89642420 0f97c00b 44242489 ...D.d$ ....D$$. - 07e0 c5894424 244489e8 488d5008 4883fa0f ..D$$D..H.P.H... - 07f0 0f867a05 00004c8d 73084585 ed0f85f5 ..z...L.s.E..... - 0800 03000049 39de0f87 af000000 8b742428 ...I9........t$( - 0810 8b7c242c 4c89f08b 4c243044 8b542434 .|$,L...L$0D.T$4 - 0820 691077ca eb854883 c01001d6 6950f477 i.w...H.....iP.w - 0830 caeb85c1 c60d69f6 b179379e 01d76950 ......i..y7...iP - 0840 f877caeb 85c1c70d 69ffb179 379e01d1 .w......i..y7... - 0850 6950fc77 caeb85c1 c10d69c9 b179379e iP.w......i..y7. - 0860 4401d2c1 c20d4469 d2b17937 9e4839d8 D.....Di..y7.H9. - 0870 76ae894c 2418660f 6e4c2418 4889d844 v..L$.f.nL$.H..D - 0880 89542418 660f6e74 24184c29 f0897424 .T$.f.nt$.L)..t$ - 0890 18660f6e 44241848 83e0f089 7c241866 .f.nD$.H....|$.f - 08a0 0f6e7c24 18660f62 ce4d8d74 0610660f .n|$.f.b.M.t..f. - 08b0 62c7660f 6cc10f11 44242848 8d531045 b.f.l...D$(H.S.E - 08c0 31ed4939 d60f838d fdffff48 8b442408 1.I9.......H.D$. - 08d0 4c29f24c 89f64989 d7488d78 184589fd L).L..I..H.x.E.. - 08e0 e8000000 0044897c 2448e969 fdffff90 .....D.|$H.i.... - 08f0 488b4308 48894414 388b4424 48448d68 H.C.H.D.8.D$HD.h - 0900 0844896c 2448662e 0f1f8400 00000000 .D.l$Hf......... - 0910 8b6c2424 448b6424 20e93afd ffff6690 .l$$D.d$ .:...f. - 0920 8b442420 8b542448 4c8d7308 4c8d7b10 .D$ .T$HL.s.L.{. - 0930 83c00848 8d4a0883 f80f8944 24200f97 ...H.J.....D$ .. - 0940 c00fb6c0 09442424 4889d048 83f90f76 .....D$$H..H...v - 0950 9f85d20f 85070200 004939de 0f87bc00 .........I9..... - 0960 00008b54 24288b7c 242c4c89 f0448b44 ...T$(.|$,L..D.D - 0970 2430448b 4c243466 0f1f8400 00000000 $0D.L$4f........ - 0980 690877ca eb854883 c0106970 f877caeb i.w...H...ip.w.. - 0990 8501ca69 48f477ca eb854401 c6c1c20d ...iH.w...D..... - 09a0 c1c60d69 d2b17937 9e01cf69 48fc77ca ...i..y7...iH.w. - 09b0 eb85c1c7 0d4469c6 b179379e 69ffb179 .....Di..y7.i..y - 09c0 379e4401 c9c1c10d 4469c9b1 79379e48 7.D.....Di..y7.H - 09d0 39d876ac 44894424 18660f6e 4c241848 9.v.D.D$.f.nL$.H - 09e0 89d84489 4c241866 0f6e7424 184c29f0 ..D.L$.f.nt$.L). - 09f0 89542418 660f6e44 24184883 e0f0897c .T$.f.nD$.H....| - 0a00 2418660f 6e7c2418 660f62ce 4d8d7406 $.f.n|$.f.b.M.t. - 0a10 10660f62 c7660f6c c10f1144 24284d39 .f.b.f.l...D$(M9 - 0a20 fe0f8209 01000044 8b6c2448 8b6c2424 .......D.l$H.l$$ - 0a30 448b6424 20e91efc ffff660f 1f440000 D.d$ .....f..D.. - 0a40 488b7424 08bd1000 000089ea 488d7c06 H.t$........H.|. - 0a50 184429ea 4c89fee8 00000000 69442438 .D).L.......iD$8 - 0a60 77caeb85 4189e803 442428c1 c00d442b w...A...D$(...D+ - 0a70 442448c7 44244800 00000069 c0b17937 D$H.D$H....i..y7 - 0a80 9e4b8d34 07894424 28694424 3c77caeb .K.4..D$(iD$<w.. - 0a90 85034424 2cc1c00d 69c0b179 379e8944 ..D$,...i..y7..D - 0aa0 242c6944 244077ca eb850344 2430c1c0 $,[email protected]$0.. - 0ab0 0d69c0b1 79379e89 44243069 44244477 .i..y7..D$0iD$Dw - 0ac0 caeb8503 442434c1 c00d69c0 b179379e ....D$4...i..y7. - 0ad0 89442434 e965f9ff ff0f1f80 00000000 .D$4.e.......... - 0ae0 4829f049 89c6488b 4424084c 89f24589 H).I..H.D$.L..E. - 0af0 f5488d78 18e80000 00004489 742448e9 .H.x......D.t$H. - 0b00 10faffff 0f1f4000 8b542428 8b44242c [email protected]$(.D$, - 0b10 c1c60cc1 c007d1c2 01c28d04 328b5424 ............2.T$ - 0b20 34c1ca0e 01d0e94e fbffff0f 1f440000 4......N.....D.. - 0b30 488b4424 084d29f7 4c89f64c 89fa4589 H.D$.M).L..L..E. - 0b40 fd488d78 18e80000 00004489 7c24488b .H.x......D.|$H. - 0b50 6c242444 8b642420 e9fbfaff ff0f1f00 l$$D.d$ ........ - 0b60 488b7424 08bd1000 0000488d 7c161889 H.t$......H.|... - 0b70 ea4c89f6 29c2e800 00000069 44243877 .L..)......iD$8w - 0b80 caeb8589 e9034424 28c1c00d 2b4c2448 ......D$(...+L$H - 0b90 c7442448 00000000 69c0b179 379e4901 .D$H....i..y7.I. - 0ba0 ce894424 28694424 3c77caeb 85034424 ..D$(iD$<w....D$ - 0bb0 2cc1c00d 69c0b179 379e8944 242c6944 ,...i..y7..D$,iD - 0bc0 244077ca eb850344 2430c1c0 0d69c0b1 [email protected]$0...i.. - 0bd0 79379e89 44243069 44244477 caeb8503 y7..D$0iD$Dw.... - 0be0 442434c1 c00d69c0 b179379e 89442434 D$4...i..y7..D$4 - 0bf0 e964fdff ff0f1f00 488b7424 0841bf10 .d......H.t$.A.. - 0c00 00000044 89fa488d 7c061844 29ea4c89 ...D..H.|..D).L. - 0c10 f6e80000 0000442b 7c24488b 6c2424c7 ......D+|$H.l$$. - 0c20 44244800 00000069 44243877 caeb8503 D$H....iD$8w.... - 0c30 4424284d 01fec1c0 0d448b64 242069c0 D$(M.....D.d$ i. - 0c40 b179379e 89442428 6944243c 77caeb85 .y7..D$(iD$<w... - 0c50 0344242c c1c00d69 c0b17937 9e894424 .D$,...i..y7..D$ - 0c60 2c694424 4077caeb 85034424 30c1c00d ,[email protected]$0... - 0c70 69c0b179 379e8944 24306944 244477ca i..y7..D$0iD$Dw. - 0c80 eb850344 2434c1c0 0d69c0b1 79379e89 ...D$4...i..y7.. - 0c90 442434e9 6bfbffff 0f1f8400 00000000 D$4.k........... - 0ca0 488b4424 084d29f7 4c89f64c 89fa4589 H.D$.M).L..L..E. - 0cb0 fd488d78 18e80000 00004489 7c2448e9 .H.x......D.|$H. - 0cc0 94f9ffff 0f1f4000 488b7424 08bd1000 [email protected]$.... - 0cd0 000089ea 488d7c06 184429ea 4c89f6e8 ....H.|..D).L... - 0ce0 00000000 69442438 77caeb85 03442428 ....iD$8w....D$( - 0cf0 4189e8c1 c00d442b 4424488b 6c2424c7 A.....D+D$H.l$$. - 0d00 44244800 00000069 c0b17937 9e4d01c6 D$H....i..y7.M.. - 0d10 448b6424 20894424 28694424 3c77caeb D.d$ .D$(iD$<w.. - 0d20 85034424 2cc1c00d 69c0b179 379e8944 ..D$,...i..y7..D - 0d30 242c6944 244077ca eb850344 2430c1c0 $,[email protected]$0.. - 0d40 0d69c0b1 79379e89 44243069 44244477 .i..y7..D$0iD$Dw - 0d50 caeb8503 442434c1 c00d69c0 b179379e ....D$4...i..y7. - 0d60 89442434 e920f8ff ff0f1f80 00000000 .D$4. .......... - 0d70 488b5308 48895404 38e97bfb ffff6690 H.S.H.T.8.{...f. - 0d80 488b7424 084c89d2 4c895424 18488d7c H.t$.L..L.T$.H.| - 0d90 06184c89 f6e80000 0000448b 6c24484c ..L.......D.l$HL - 0da0 8b542418 8b6c2424 448b6424 204501d5 .T$..l$$D.d$ E.. - 0db0 44896c24 48e99ef8 ffff660f 1f440000 D.l$H.....f..D.. - 0dc0 8b742430 8b6c2424 488d4424 20488944 .t$0.l$$H.D$ H.D - 0dd0 2408448b 6424208d 86b16756 1685ed0f $.D.d$ ...gV.... - 0de0 8494f8ff ffe91efd ffff660f 1f440000 ..........f..D.. - 0df0 31d2e900 00000066 0f1f8400 00000000 1......f........ - 0e00 41544989 f4554889 fdbf1800 000053e8 ATI..UH.......S. - 0e10 00000000 4d85e44c 89e64889 c3b80000 ....M..L..H..... - 0e20 0000480f 44e84c89 6308c643 14004889 ..H.D.L.c..C..H. - 0e30 ef48892b e8000000 00894310 4889d85b .H.+......C.H..[ - 0e40 5d415cc3 66662e0f 1f840000 00000090 ]A\.ff.......... - 0e50 807f1400 750d488b 4708c647 14014885 ....u.H.G..G..H. - 0e60 c07505c3 0f1f4000 41554154 4531e455 [email protected] - 0e70 4889fd48 8d3c4053 48c1e703 31db4883 H..H.<@SH...1.H. - 0e80 ec08e800 00000049 89c5660f 1f440000 .......I..f..D.. - 0e90 488b7500 498d7c1d 004983c4 014801de H.u.I.|..I...H.. - 0ea0 4883c318 e8000000 004c3965 0877e14c H........L9e.w.L - 0eb0 896d0048 83c4085b 5d415c41 5dc36690 .m.H...[]A\A].f. - 0ec0 488b4708 48890248 85c0750c 48c70600 H.G.H..H..u.H... - 0ed0 000000c3 0f1f4000 41554154 4c8d2440 [email protected].$@ - 0ee0 5549c1e4 034889fd 534c89e7 4889f348 UI...H..SL..H..H - 0ef0 83ec08e8 00000000 48890349 89c531db ........H..I..1. - 0f00 488b7500 498d7c1d 004801de 4883c318 H.u.I.|..H..H... - 0f10 e8000000 004939dc 75e64883 c4085b5d .....I9.u.H...[] - 0f20 415c415d c366662e 0f1f8400 00000000 A\A].ff......... - 0f30 41545548 89fdbf18 00000053 e8000000 ATUH.......S.... - 0f40 008b5510 4989c448 8b450841 89542410 ..U.I..H.E.A.T$. - 0f50 49894424 0841c644 24140148 85c07510 I.D$.A.D$..H..u. - 0f60 4c89e05b 5d49c704 24000000 00415cc3 L..[]I..$....A\. - 0f70 488d3c40 31db48c1 e703e800 00000049 H.<@1.H........I - 0f80 890424eb 070f1f00 498b0424 488d3c5b ..$.....I..$H.<[ - 0f90 488b7500 4883c301 48c1e703 4801fe48 H.u.H...H...H..H - 0fa0 01c7e800 00000049 395c2408 77da4c89 .......I9\$.w.L. - 0fb0 e05b5d41 5cc3662e 0f1f8400 00000000 .[]A\.f......... - 0fc0 53807f14 004889fb 750e4889 df5be900 S....H..u.H..[.. - 0fd0 0000000f 1f440000 488b7708 488b3fe8 .....D..H.w.H.?. - 0fe0 00000000 4889df5b e9000000 000f1f00 ....H..[........ - 0ff0 89f2488b 7708488b 3fe90000 00006690 ..H.w.H.?.....f. - 1000 41574156 41554154 55534883 ec584889 AWAVAUATUSH..XH. - 1010 54244848 894c2440 4885f60f 84a80200 T$HH.L$@H....... - 1020 004889fd bf000400 004889f3 e8000000 .H.......H...... - 1030 00bf0004 00004989 c5e80000 00004989 ......I.......I. - 1040 c485db0f 8ee80200 008d43ff 48896c24 ..........C.H.l$ - 1050 0841be00 040000bb 00040000 488d0440 .A..........H..@ - 1060 c7442424 00000000 48c74424 10000400 .D$$....H.D$.... - 1070 00488d44 c5184889 4424180f 1f440000 .H.D..H.D$...D.. - 1080 488b4424 088b4008 83f8010f 84ef0100 H.D$..@......... - 1090 0085c00f 84770100 0083f802 0f84de00 .....w.......... - 10a0 00004863 6c24244c 8b7c2410 498d8600 ..Hcl$$L.|$.I... - 10b0 0400004c 29f84889 c24c39fd 72334c89 ...L).H..L9.r3L. - 10c0 e04989dc 4889d366 0f1f8400 00000000 .I..H..f........ - 10d0 4e8d343b 4889c749 81c70004 00004c89 N.4;H..I......L. - 10e0 f6e80000 00004939 ef76e54c 89e34989 ......I9.v.L..I. - 10f0 c44c89f7 4889ea4c 89ee4c29 ff4c01e7 .L..H..L..L).L.. - 1100 e8000000 004c89f8 48834424 08184829 .....L..H.D$..H) - 1110 e8488944 2410488b 44240848 39442418 .H.D$.H.D$.H9D$. - 1120 0f855aff ffff4c29 fd4c01f5 488d7d01 ..Z...L).L..H.}. - 1130 e8000000 004889ea 4c89e648 89c1488b .....H..L..H..H. - 1140 44244848 89cf4889 08e80000 00004c89 D$HH..H.......L. - 1150 e7c60428 00488b44 24404889 28e80000 ...(.H.D$@H.(... - 1160 00004883 c4584c89 ef5b5d41 5c415d41 ..H..XL..[]A\A]A - 1170 5e415fe9 00000000 0f1f8400 00000000 ^A_............. - 1180 488b6c24 08488b45 004889c7 48894424 H.l$.H.E.H..H.D$ - 1190 28e80000 00004989 c7488b45 104889c7 (.....I..H.E.H.. - 11a0 48894424 30e80000 00004889 44243844 H.D$0.....H.D$8D - 11b0 01f84863 e8894424 244839dd 72244c89 ..Hc..D$$H9.r$L. - 11c0 e80f1f80 00000000 4881c300 04000048 ........H......H - 11d0 89c74889 dee80000 00004839 eb76e949 ..H.......H9.v.I - 11e0 89c5488b 7424284c 89fa4c89 efe80000 ..H.t$(L..L..... - 11f0 0000488b 54243848 8b742430 4b8d7c3d ..H.T$8H.t$0K.|= - 1200 00e80000 0000e99c feffff0f 1f440000 .............D.. - 1210 488b4424 08488b00 4889c748 89442428 H.D$.H..H..H.D$( - 1220 e8000000 004989c7 8d400848 63e88944 [email protected] - 1230 24244839 dd72234c 89e8660f 1f440000 $$H9.r#L..f..D.. - 1240 4881c300 04000048 89c74889 dee80000 H......H..H..... - 1250 00004839 eb76e949 89c5488b 7424284c ..H9.v.I..H.t$(L - 1260 89fa4c89 efe80000 0000488b 44240848 ..L.......H.D$.H - 1270 8b40104b 89443d00 e92afeff ff0f1f00 [email protected]=..*...... - 1280 488b4424 08488b00 4889c748 89442428 H.D$.H..H..H.D$( - 1290 e8000000 004989c7 8d400848 63e88944 [email protected] - 12a0 24244839 dd72b34c 89e8660f 1f440000 $$H9.r.L..f..D.. - 12b0 4881c300 04000048 89c74889 dee80000 H......H..H..... - 12c0 00004839 eb76e9eb 8e488d3d 00000000 ..H9.v...H.=.... - 12d0 e8000000 00488b54 24484889 c1488902 .....H.T$HH..H.. - 12e0 8b314883 c1048d96 fffefefe f7d621f2 .1H...........!. - 12f0 81e28080 808074e8 89d6c1ee 10f7c280 ......t......... - 1300 8000000f 44d6488d 7102480f 44ce89d6 ....D.H.q.H.D... - 1310 4000d648 83d90348 29c1488b 44244048 @..H...H).H.D$@H - 1320 89084883 c4585b5d 415c415d 415e415f ..H..X[]A\A]A^A_ - 1330 c3bf0100 000031ed e9f3fdff ff0f1f00 ......1......... - 1340 4883ec18 488b7708 488b3f48 8d4c2408 H...H.w.H.?H.L$. - 1350 4889e2e8 00000000 488b0424 4883c418 H.......H..$H... - 1360 c3 . -Contents of section .rodata.str1.1: - 0000 6e6f2074 61677300 no tags. -Contents of section .debug_info: - 0000 e4260000 04000000 00000801 00000000 .&.............. - 0010 0c000000 00000000 00000000 00000000 ................ - 0020 00611300 00000000 00000000 00020805 .a.............. - 0030 00000000 03000000 0003d817 45000000 ............E... - 0040 04340000 00020807 00000000 05040569 .4.............i - 0050 6e740002 08050000 00000210 04000000 nt.............. - 0060 00020407 00000000 06080768 00000002 ...........h.... - 0070 01080000 00000202 07000000 00020106 ................ - 0080 00000000 03000000 00042517 6f000000 ..........%.o... - 0090 02020500 00000003 00000000 04291661 .............).a - 00a0 00000003 00000000 0496192d 00000003 ...........-.... - 00b0 00000000 04971b2d 00000008 08c10000 .......-........ - 00c0 00020106 00000000 04c10000 00090000 ................ - 00d0 0000d805 31085402 00000a00 00000005 ....1.T......... - 00e0 33074c00 0000000a 00000000 053609bb 3.L..........6.. - 00f0 00000008 0a000000 00053709 bb000000 ..........7..... - 0100 100a0000 00000538 09bb0000 00180a00 .......8........ - 0110 00000005 3909bb00 0000200a 00000000 ....9..... ..... - 0120 053a09bb 00000028 0a000000 00053b09 .:.....(......;. - 0130 bb000000 300a0000 0000053c 09bb0000 ....0......<.... - 0140 00380a00 00000005 3d09bb00 0000400a .8......=.....@. - 0150 00000000 054009bb 00000048 0a000000 [email protected].... - 0160 00054109 bb000000 500a0000 00000542 ..A.....P......B - 0170 09bb0000 00580a00 00000005 44166d02 .....X......D.m. - 0180 0000600a 00000000 05461473 02000068 ..`......F.s...h - 0190 0a000000 00054807 4c000000 700a0000 ......H.L...p... - 01a0 00000549 074c0000 00740a00 00000005 ...I.L...t...... - 01b0 4a0ba300 0000780a 00000000 054d1276 J.....x......M.v - 01c0 00000080 0a000000 00054e0f 7d000000 ..........N.}... - 01d0 820a0000 0000054f 08790200 00830a00 .......O.y...... - 01e0 00000005 510f8902 0000880a 00000000 ....Q........... - 01f0 05590daf 00000090 0a000000 00055b17 .Y............[. - 0200 94020000 980a0000 0000055c 199f0200 ...........\.... - 0210 00a00a00 00000005 5d147302 0000a80a ........].s..... - 0220 00000000 055e0968 000000b0 0a000000 .....^.h........ - 0230 00055f0a 34000000 b80a0000 00000560 .._.4..........` - 0240 074c0000 00c00a00 00000005 6208a502 .L..........b... - 0250 0000c400 03000000 00060719 cd000000 ................ - 0260 0b000000 00052b0e 0c000000 00080868 ......+........h - 0270 02000008 08cd0000 000dc100 00008902 ................ - 0280 00000e45 00000000 00080860 0200000c ...E.......`.... - 0290 00000000 08088f02 00000c00 00000008 ................ - 02a0 089a0200 000dc100 0000b502 00000e45 ...............E - 02b0 00000013 000f0000 00000789 0ec10200 ................ - 02c0 00080854 0200000f 00000000 078a0ec1 ...T............ - 02d0 0200000f 00000000 078b0ec1 0200000f ................ - 02e0 00000000 081a0c4c 0000000d 01030000 .......L........ - 02f0 f6020000 100004eb 02000008 08c80000 ................ - 0300 0004fb02 00000f00 00000008 1b1af602 ................ - 0310 00000208 07000000 00080824 03000007 ...........$.... - 0320 19030000 11120704 61000000 0257010e ........a....W.. - 0330 41030000 13000000 00001300 00000001 A............... - 0340 00140000 00000257 01262503 00000300 .......W.&%..... - 0350 00000009 18138400 00000300 00000009 ................ - 0360 1a149700 00001400 00000002 6901165a ............i..Z - 0370 03000014 00000000 02d9011e 85030000 ................ - 0380 04730300 00150000 00003002 cf0308e6 .s........0..... - 0390 03000016 00000000 02d00311 66030000 ............f... - 03a0 00160000 000002d1 03116603 00000417 ..........f..... - 03b0 760002d2 0311e603 00000816 00000000 v............... - 03c0 02d30311 e6030000 18160000 000002d4 ................ - 03d0 03116603 00002816 00000000 02d50311 ..f...(......... - 03e0 66030000 2c000d66 030000f6 0300000e f...,..f........ - 03f0 45000000 03001400 00000002 3406134e E...........4..N - 0400 03000004 f6030000 14000000 00023806 ..............8. - 0410 16660300 00040804 00001207 04610000 .f...........a.. - 0420 00021b07 0e360400 00130000 00000013 .....6.......... - 0430 00000000 01001400 00000002 1e07031a ................ - 0440 04000002 04040000 00000208 04000000 ................ - 0450 000d0304 00006104 00000e45 000000bf ......a....E.... - 0460 00045104 00001800 00000002 650c2361 ..Q.........e.#a - 0470 04000040 c0b8fe6c 3923a44b be7c0181 [email protected]#.K.|.. - 0480 2cf721ad 1cded46d e9839097 db7240a4 ,.!....m.....r@. - 0490 a4b7b367 1fcb79e6 4eccc0e5 78825ad0 ...g..y.N...x.Z. - 04a0 7dccff72 21b80846 74f74324 8ee03590 }..r!..Ft.C$..5. - 04b0 e6813a26 4c3c2852 bb91c300 cb88d065 ..:&L<(R.......e - 04c0 8b1b532e a3716448 97a20df9 4e3819ef ..S..qdH....N8.. - 04d0 46a9deac d8a8fa76 3fe39c34 3ff9dcbb F......v?..4?... - 04e0 c7c70b4f 1d8a51e0 4bcdb459 31c89f7e ...O..Q.K..Y1..~ - 04f0 c9d97873 64eac5ac 8334d3eb c3c581a0 ..xsd....4...... - 0500 fffa1363 eb170ddd 51b7f0da 49d31655 ...c....Q...I..U - 0510 2629d468 9e2b16be 587d47a1 fc8ff8b8 &).h.+..X}G..... - 0520 d17ad031 ce45cb3a 8f951604 28afd7fb .z.1.E.:....(... - 0530 cabb4b40 7e080803 04000007 35050000 ..K@~.......5... - 0540 04350500 00190000 00000704 61000000 .5..........a... - 0550 0a1b066a 05000013 00000000 00130000 ...j............ - 0560 00000113 00000000 02001a08 0a2a0298 .............*.. - 0570 0500001b 00000000 0a2b0d53 0000001b .........+.S.... - 0580 00000000 0a2c0a4a 0400001b 00000000 .....,.J........ - 0590 0a2d0ffb 02000000 09000000 00180a27 .-.............' - 05a0 08c60500 001c6b65 79000a28 0efb0200 ......key..(.... - 05b0 00000a00 0000000a 29164505 0000081d ........).E..... - 05c0 6a050000 10000498 05000008 08980500 j............... - 05d0 00090000 00001801 0f081306 00000a00 ................ - 05e0 00000001 111bcb05 0000000a 00000000 ................ - 05f0 01120c34 00000008 0a000000 00011312 ...4............ - 0600 61000000 100a0000 00000114 0a180600 a............... - 0610 00140004 d1050000 02010200 00000009 ................ - 0620 00000000 04018908 3a060000 0a000000 ........:....... - 0630 00018a12 61000000 0000041f 0600001e ....a........... - 0640 00000000 01850107 bb000000 00000000 ................ - 0650 00000000 21000000 00000000 019cb206 ....!........... - 0660 00001f74 61670001 850140b2 06000000 ...tag....@..... - 0670 00000000 00000020 6b657900 0187010b ....... key..... - 0680 bb000000 02916021 00000000 0188010c ......`!........ - 0690 34000000 02916822 00000000 00000000 4.....h"........ - 06a0 b8060000 23015102 77002301 52029168 ....#.Q.w.#.R..h - 06b0 00000808 13060000 24000000 00013201 ........$.....2. - 06c0 06018807 00002500 00000001 32013888 ......%.....2.8. - 06d0 07000025 00000000 01320147 34000000 ...%.....2.G4... - 06e0 25000000 00013201 568e0700 00250000 %.....2.V....%.. - 06f0 00000132 01679407 00002669 00013a01 ...2.g....&i..:. - 0700 064c0000 00270000 0000013b 01064c00 .L...'.....;..L. - 0710 00002674 61670001 3c0118cb 05000027 ..&tag..<......' - 0720 00000000 013e0112 40000000 27000000 .....>..@...'... - 0730 00013f01 0c340000 00270000 00000140 ..?..4...'.....@ - 0740 010bbb00 00002700 00000001 41010c34 ......'.....A..4 - 0750 00000027 00000000 0142010c 34000000 ...'.....B..4... - 0760 27000000 00014301 0bbb0000 00270000 '.....C......'.. - 0770 00000144 010c3400 00002700 00000001 ...D..4...'..... - 0780 45010c34 00000000 0808c605 00000808 E..4............ - 0790 bb000000 08083400 00001e00 00000001 ......4......... - 07a0 2c010a61 00000000 00000000 0000000e ,..a............ - 07b0 00000000 00000001 9cfd0700 001f7461 ..............ta - 07c0 6700012c 013fb206 00000000 00000000 g..,.?.......... - 07d0 00002800 00000001 2c014d61 00000000 ..(.....,.Ma.... - 07e0 00000000 00000029 00000000 00000000 .......)........ - 07f0 810b0000 23015103 f3015400 002a0000 ....#.Q...T..*.. - 0800 00000123 01060000 00000000 00002d00 ...#..........-. - 0810 00000000 0000019c 6d080000 28000000 ........m...(... - 0820 00012301 2d6d0800 00000000 00000000 ..#.-m.......... - 0830 002b0000 00000000 00009026 00004a08 .+.........&..J. - 0840 00002301 5503f301 55002c00 00000000 ..#.U...U.,..... - 0850 0000006a 1e000029 00000000 00000000 ...j...)........ - 0860 90260000 23015503 f3015500 000808d1 .&..#.U...U..... - 0870 0500001e 00000000 01100116 6d080000 ............m... - 0880 00000000 00000000 86000000 00000000 ................ - 0890 019c1b09 00001f73 72630001 100143b2 .......src....C. - 08a0 06000000 00000000 0000002d 64737400 ...........-dst. - 08b0 0112011a 6d080000 00000000 00000000 ....m........... - 08c0 2e000000 00000000 002b0000 00000000 .........+...... - 08d0 00f60800 002d6900 011b0112 4c000000 .....-i.....L... - 08e0 00000000 00000000 2c000000 00000000 ........,....... - 08f0 00aa1e00 00002f00 00000000 0000009d ....../......... - 0900 2600000d 09000023 01550148 002c0000 &......#.U.H.,.. - 0910 00000000 00009d26 00000030 00000000 .......&...0.... - 0920 01ff0600 00000000 00000065 00000000 ...........e.... - 0930 00000001 9cea0900 00310000 000001ff .........1...... - 0940 47b20600 00000000 00000000 00326f75 G............2ou - 0950 740001ff 67ea0900 00000000 00000000 t...g........... - 0960 00310000 000001ff 74940700 00000000 .1......t....... - 0970 00000000 00330000 00000101 010c3400 .....3........4. - 0980 00000000 00000000 00003300 00000001 ..........3..... - 0990 08011bcb 05000000 00000000 00000034 ...............4 - 09a0 00000000 d5090000 2d690001 0b010e4c ........-i.....L - 09b0 00000000 00000000 00000022 00000000 ...........".... - 09c0 00000000 aa1e0000 23015507 7d007300 ........#.U.}.s. - 09d0 22481c00 00220000 00000000 00009d26 "H...".........& - 09e0 00002301 55027c00 00000808 cb050000 ..#.U.|......... - 09f0 30000000 0001ee06 00000000 00000000 0............... - 0a00 6e000000 00000000 019c850a 00003100 n.............1. - 0a10 00000001 ee2e6d08 00000000 00000000 ......m......... - 0a20 00003500 00000001 f71bcb05 00000000 ..5............. - 0a30 00000000 00002e00 00000000 00000025 ...............% - 0a40 00000000 00000077 0a000036 690001f8 .......w...6i... - 0a50 0e4c0000 00000000 00000000 00220000 .L...........".. - 0a60 00000000 0000aa1e 00002301 55077d00 ..........#.U.}. - 0a70 73002248 1c00002c 00000000 00000000 s."H...,........ - 0a80 9d260000 00370000 000001de 166d0800 .&...7.......m.. - 0a90 00000000 00000000 00440000 00000000 .........D...... - 0aa0 00019c15 0b000032 73726300 01de5c88 .......2src...\. - 0ab0 07000000 00000000 00000031 00000000 ...........1.... - 0ac0 01de6834 00000000 00000000 00000035 ..h4...........5 - 0ad0 00000000 01e01a6d 08000000 00000000 .......m........ - 0ae0 0000002f 00000000 00000000 9d260000 .../.........&.. - 0af0 fa0a0000 23015501 48002200 00000000 ....#.U.H."..... - 0b00 00000015 0b000023 01550276 00230154 .......#.U.v.#.T - 0b10 027c0000 00370000 000001d9 0e610000 .|...7.......a.. - 0b20 00000000 00000000 00070000 00000000 ................ - 0b30 00019c81 0b000031 00000000 01d935cb .......1......5. - 0b40 05000000 00000000 00000031 00000000 ...........1.... - 0b50 01d94234 00000000 00000000 00000029 ..B4...........) - 0b60 00000000 00000000 810b0000 23015503 ............#.U. - 0b70 f3015523 015403f3 01542301 51013000 ..U#.T...T#.Q.0. - 0b80 00370000 000001bd 0e610000 00000000 .7.......a...... - 0b90 00000000 009a0a00 00000000 00019c76 ...............v - 0ba0 1b000032 74616700 01bd38cb 05000000 ...2tag...8..... - 0bb0 00000000 00000031 00000000 01bd4434 .......1......D4 - 0bc0 00000000 00000000 00000031 00000000 ...........1.... - 0bd0 01bd5861 00000000 00000000 00000038 ..Xa...........8 - 0be0 00000000 01bf1373 03000003 91e07e34 .......s......~4 - 0bf0 00000000 141a0000 36690001 c20e4c00 ........6i....L. - 0c00 00000000 00000000 000039d3 1f000000 ..........9..... - 0c10 00000000 00000000 00000000 0001c409 ................ - 0c20 970f0000 3aff1f00 00000000 00000000 ....:........... - 0c30 003af21f 00000000 00000000 00003ae5 .:............:. - 0c40 1f000000 00000000 0000003b 0c200000 ...........;. .. - 0c50 00000000 3c0d2000 00000000 00000000 ....<. ......... - 0c60 003c1820 00000000 00000000 00003d38 .<. ..........=8 - 0c70 20000000 0000003d 0d00003e 39200000 ......=...>9 .. - 0c80 3ff62000 00000000 00000000 000a0000 ?. ............. - 0c90 00000002 a2081fb6 0c00003a 15210000 ...........:.!.. - 0ca0 00000000 00000000 3a082100 00000000 ........:.!..... - 0cb0 00000000 00003ff6 20000000 00000000 ......?. ....... - 0cc0 0000000a 00000000 0002a308 1fe40c00 ................ - 0cd0 00401521 00003a08 21000000 00000000 .@.!..:.!....... - 0ce0 00000000 3ff62000 00000000 00000000 ....?. ......... - 0cf0 000a0000 00000002 a4081f12 0d000040 ...............@ - 0d00 15210000 3a082100 00000000 00000000 .!..:.!......... - 0d10 000041f6 20000000 00000000 0000000a ..A. ........... - 0d20 00000000 0002a508 1f401521 00003a08 .........@.!..:. - 0d30 21000000 00000000 00000000 003f9d21 !............?.! - 0d40 00000000 00000000 00000200 00000000 ................ - 0d50 028d080d ad0d0000 3ac92100 00000000 ........:.!..... - 0d60 00000000 003abc21 00000000 00000000 .....:.!........ - 0d70 00003aaf 21000000 00000000 00000022 ..:.!.........." - 0d80 00000000 00000000 aa260000 2301550f .........&..#.U. - 0d90 7d000cff ffffff1a 91c87e06 22231823 }.........~."#.# - 0da0 0154027f 00230151 027e0000 003f9d21 .T...#.Q.~...?.! - 0db0 00000000 00000000 00000500 00000000 ................ - 0dc0 0293080d 250e0000 3ac92100 00000000 ....%...:.!..... - 0dd0 00000000 003abc21 00000000 00000000 .....:.!........ - 0de0 00003aaf 21000000 00000000 00000022 ..:.!.........." - 0df0 00000000 00000000 aa260000 2301550f .........&..#.U. - 0e00 7d000cff ffffff1a 91c87e06 22231823 }.........~."#.# - 0e10 0154027f 00230151 0a407d00 1c0cffff .T...#.Q.@}..... - 0e20 ffff1a00 003d2520 00000000 0000380f .....=% ......8. - 0e30 00003c2a 20000000 00000000 0000003f ..<* ..........? - 0e40 f6200000 00000000 00000000 09000000 . .............. - 0e50 00000295 081f750e 00003a15 21000000 ......u...:.!... - 0e60 00000000 0000003a 08210000 00000000 .......:.!...... - 0e70 00000000 0042f620 00000000 00000000 .....B. ........ - 0e80 00000800 00000000 00000000 15000000 ................ - 0e90 00000000 0296081f b70e0000 3a152100 ............:.!. - 0ea0 00000000 00000000 003a0821 00000000 .........:.!.... - 0eb0 00000000 00000042 f6200000 00000000 .......B. ...... - 0ec0 00000000 08000000 00000000 00001500 ................ - 0ed0 00000000 00000297 081ff90e 00003a15 ..............:. - 0ee0 21000000 00000000 0000003a 08210000 !..........:.!.. - 0ef0 00000000 00000000 0043f620 00000000 .........C. .... - 0f00 00000000 00000800 00000000 00000000 ................ - 0f10 15000000 00000000 0298081f 3a152100 ............:.!. - 0f20 00000000 00000000 003a0821 00000000 .........:.!.... - 0f30 00000000 00000000 419d2100 00000000 ........A.!..... - 0f40 00000000 00000000 00000002 ab080d3a ...............: - 0f50 c9210000 00000000 00000000 3abc2100 .!..........:.!. - 0f60 00000000 00000000 003aaf21 00000000 .........:.!.... - 0f70 00000000 00002200 00000000 000000aa ......"......... - 0f80 26000023 01550691 c87e0623 18230151 &..#.U...~.#.#.Q - 0f90 027e0000 00000039 d31f0000 00000000 .~.....9........ - 0fa0 00000000 00000000 000001ce 0d2c1300 .............,.. - 0fb0 003aff1f 00000000 00000000 00003af2 .:............:. - 0fc0 1f000000 00000000 0000003a e51f0000 ...........:.... - 0fd0 00000000 00000000 3b0c2000 00000000 ........;. ..... - 0fe0 003c0d20 00000000 00000000 00003c18 .<. ..........<. - 0ff0 20000000 00000000 0000003d 38200000 ..........=8 .. - 1000 00000000 ca100000 3e392000 003ff620 ........>9 ..?. - 1010 00000000 00000000 00000a00 00000000 ................ - 1020 02a2081f 43100000 3a152100 00000000 ....C...:.!..... - 1030 00000000 003a0821 00000000 00000000 .....:.!........ - 1040 0000003f f6200000 00000000 00000000 ...?. .......... - 1050 0a000000 000002a3 081f7110 00004015 ..........q...@. - 1060 2100003a 08210000 00000000 00000000 !..:.!.......... - 1070 003ff620 00000000 00000000 00000a00 .?. ............ - 1080 00000000 02a4081f 9f100000 40152100 ............@.!. - 1090 003a0821 00000000 00000000 00000041 .:.!...........A - 10a0 f6200000 00000000 00000000 0a000000 . .............. - 10b0 000002a5 081f4015 2100003a 08210000 ......@.!..:.!.. - 10c0 00000000 00000000 00003f9d 21000000 ..........?.!... - 10d0 00000000 00000000 00000000 0002ab08 ................ - 10e0 0d311100 003ac921 00000000 00000000 .1...:.!........ - 10f0 00003abc 21000000 00000000 0000003a ..:.!..........: - 1100 af210000 00000000 00000000 22000000 .!.........."... - 1110 00000000 00aa2600 00230155 0691c87e ......&..#.U...~ - 1120 06231823 0154027e 00230151 027f0000 .#.#.T.~.#.Q.... - 1130 003f9d21 00000000 00000000 00000400 .?.!............ - 1140 00000000 0293080d a9110000 3ac92100 ............:.!. - 1150 00000000 00000000 003abc21 00000000 .........:.!.... - 1160 00000000 00003aaf 21000000 00000000 ......:.!....... - 1170 00000022 00000000 00000000 aa260000 ...".........&.. - 1180 2301550f 7d000cff ffffff1a 91c87e06 #.U.}.........~. - 1190 22231823 0154027e 00230151 0a407d00 "#.#.T.~.#.Q.@}. - 11a0 1c0cffff ffff1a00 003d2520 00000000 .........=% .... - 11b0 0000bc12 00003c2a 20000000 00000000 ......<* ....... - 11c0 0000003f f6200000 00000000 00000000 ...?. .......... - 11d0 09000000 00000295 081ff911 00003a15 ..............:. - 11e0 21000000 00000000 0000003a 08210000 !..........:.!.. - 11f0 00000000 00000000 0042f620 00000000 .........B. .... - 1200 00000000 00000800 00000000 00000000 ................ - 1210 15000000 00000000 0296081f 3b120000 ............;... - 1220 3a152100 00000000 00000000 003a0821 :.!..........:.! - 1230 00000000 00000000 00000042 f6200000 ...........B. .. - 1240 00000000 00000000 08000000 00000000 ................ - 1250 00001500 00000000 00000297 081f7d12 ..............}. - 1260 00003a15 21000000 00000000 0000003a ..:.!..........: - 1270 08210000 00000000 00000000 0043f620 .!...........C. - 1280 00000000 00000000 00000800 00000000 ................ - 1290 00000000 15000000 00000000 0298081f ................ - 12a0 3a152100 00000000 00000000 003a0821 :.!..........:.! - 12b0 00000000 00000000 00000000 419d2100 ............A.!. - 12c0 00000000 00000000 00090000 00000002 ................ - 12d0 8d080d3a c9210000 00000000 00000000 ...:.!.......... - 12e0 3abc2100 00000000 00000000 003aaf21 :.!..........:.! - 12f0 00000000 00000000 00002200 00000000 .........."..... - 1300 000000aa 26000023 01550f7d 000cffff ....&..#.U.}.... - 1310 ffff1a91 c87e0622 23182301 54027e00 .....~."#.#.T.~. - 1320 23015104 91d87e06 00000000 39d31f00 #.Q...~.....9... - 1330 00000000 00000000 00040000 00000001 ................ - 1340 cb0d9e16 00003aff 1f000000 00000000 ......:......... - 1350 0000003a f21f0000 00000000 00000000 ...:............ - 1360 3ae51f00 00000000 00000000 003b0c20 :............;. - 1370 00000000 00003c0d 20000000 00000000 ......<. ....... - 1380 0000003c 18200000 00000000 00000000 ...<. .......... - 1390 3d382000 00000000 005f1400 003e3920 =8 ......_...>9 - 13a0 00003ff6 20000000 00000000 0000000d ..?. ........... - 13b0 00000000 0002a208 1fd81300 003a1521 .............:.! - 13c0 00000000 00000000 00003a08 21000000 ..........:.!... - 13d0 00000000 00000000 3ff62000 00000000 ........?. ..... - 13e0 00000000 000a0000 00000002 a3081f06 ................ - 13f0 14000040 15210000 3a082100 00000000 ...@.!..:.!..... - 1400 00000000 00003ff6 20000000 00000000 ......?. ....... - 1410 0000000a 00000000 0002a408 1f341400 .............4.. - 1420 00401521 00003a08 21000000 00000000 .@.!..:.!....... - 1430 00000000 41f62000 00000000 00000000 ....A. ......... - 1440 000a0000 00000002 a5081f40 15210000 ...........@.!.. - 1450 3a082100 00000000 00000000 0000003f :.!............? - 1460 9d210000 00000000 00000000 00000000 .!.............. - 1470 000002ab 080dc614 00003ac9 21000000 ..........:.!... - 1480 00000000 0000003a bc210000 00000000 .......:.!...... - 1490 00000000 3aaf2100 00000000 00000000 ....:.!......... - 14a0 00220000 00000000 0000aa26 00002301 .".........&..#. - 14b0 550691c8 7e062318 23015402 7e002301 U...~.#.#.T.~.#. - 14c0 51027f00 00003f9d 21000000 00000000 Q.....?.!....... - 14d0 00000004 00000000 00029308 0d3e1500 .............>.. - 14e0 003ac921 00000000 00000000 00003abc .:.!..........:. - 14f0 21000000 00000000 0000003a af210000 !..........:.!.. - 1500 00000000 00000000 22000000 00000000 ........"....... - 1510 00aa2600 00230155 0f7d000c ffffffff ..&..#.U.}...... - 1520 1a91c87e 06222318 23015402 7e002301 ...~."#.#.T.~.#. - 1530 510a407d 001c0cff ffffff1a 00003d25 Q.@}..........=% - 1540 20000000 00000051 1600003c 2a200000 ......Q...<* .. - 1550 00000000 00000000 3ff62000 00000000 ........?. ..... - 1560 00000000 00090000 00000002 95081f8e ................ - 1570 1500003a 15210000 00000000 00000000 ...:.!.......... - 1580 3a082100 00000000 00000000 000042f6 :.!...........B. - 1590 20000000 00000000 00000008 00000000 ............... - 15a0 00000000 00150000 00000000 00029608 ................ - 15b0 1fd01500 003a1521 00000000 00000000 .....:.!........ - 15c0 00003a08 21000000 00000000 00000000 ..:.!........... - 15d0 42f62000 00000000 00000000 00080000 B. ............. - 15e0 00000000 00000015 00000000 00000002 ................ - 15f0 97081f12 1600003a 15210000 00000000 .......:.!...... - 1600 00000000 3a082100 00000000 00000000 ....:.!......... - 1610 000043f6 20000000 00000000 00000008 ..C. ........... - 1620 00000000 00000000 00150000 00000000 ................ - 1630 00029808 1f3a1521 00000000 00000000 .....:.!........ - 1640 00003a08 21000000 00000000 00000000 ..:.!........... - 1650 00439d21 00000000 00000000 00000400 .C.!............ - 1660 00000000 00000000 10000000 00000000 ................ - 1670 028d080d 3ac92100 00000000 00000000 ....:.!......... - 1680 003abc21 00000000 00000000 00003aaf .:.!..........:. - 1690 21000000 00000000 00000000 000039d3 !.............9. - 16a0 1f000000 00000000 00000001 00000000 ................ - 16b0 0001c80d e7190000 3aff1f00 00000000 ........:....... - 16c0 00000000 003af21f 00000000 00000000 .....:.......... - 16d0 00003ae5 1f000000 00000000 0000003b ..:............; - 16e0 0c200000 00000000 3c0d2000 00000000 . ......<. ..... - 16f0 00000000 003c1820 00000000 00000000 .....<. ........ - 1700 0000429d 21000000 00000000 00000005 ..B.!........... - 1710 00000000 00000000 00090000 00000000 ................ - 1720 00028d08 0d511700 003ac921 00000000 .....Q...:.!.... - 1730 00000000 00003abc 21000000 00000000 ......:.!....... - 1740 0000003a af210000 00000000 00000000 ...:.!.......... - 1750 003d3820 00000000 00002018 00003e39 .=8 ...... ...>9 - 1760 2000003f f6200000 00000000 00000000 ..?. .......... - 1770 0d000000 000002a2 081f9917 00003a15 ..............:. - 1780 21000000 00000000 0000003a 08210000 !..........:.!.. - 1790 00000000 00000000 003ff620 00000000 .........?. .... - 17a0 00000000 00000a00 00000000 02a4081f ................ - 17b0 c7170000 40152100 003a0821 00000000 ....@.!..:.!.... - 17c0 00000000 0000003f f6200000 00000000 .......?. ...... - 17d0 00000000 0a000000 000002a3 081ff517 ................ - 17e0 00004015 2100003a 08210000 00000000 ..@.!..:.!...... - 17f0 00000000 0041f620 00000000 00000000 .....A. ........ - 1800 00001800 00000000 02a5081f 40152100 ............@.!. - 1810 003a0821 00000000 00000000 00000000 .:.!............ - 1820 3f9d2100 00000000 00000000 00000000 ?.!............. - 1830 00000002 ab080d7f 1800003a c9210000 ...........:.!.. - 1840 00000000 00000000 40bc2100 003aaf21 ........@.!..:.! - 1850 00000000 00000000 00002200 00000000 .........."..... - 1860 000000aa 26000023 01550691 c87e0623 ....&..#.U...~.# - 1870 18230154 027e0023 0151027f 0000003f .#.T.~.#.Q.....? - 1880 9d210000 00000000 00000000 03000000 .!.............. - 1890 00000293 080dd618 00003ac9 21000000 ..........:.!... - 18a0 00000000 0000003a bc210000 00000000 .......:.!...... - 18b0 00000000 3aaf2100 00000000 00000000 ....:.!......... - 18c0 00220000 00000000 0000aa26 00002301 .".........&..#. - 18d0 54027e00 00003b25 20000000 0000003c T.~...;% ......< - 18e0 2a200000 00000000 00000000 3ff62000 * ..........?. . - 18f0 00000000 00000000 00090000 00000002 ................ - 1900 95081f22 1900003a 15210000 00000000 ..."...:.!...... - 1910 00000000 3a082100 00000000 00000000 ....:.!......... - 1920 000042f6 20000000 00000000 00000008 ..B. ........... - 1930 00000000 00000000 00150000 00000000 ................ - 1940 00029608 1f641900 003a1521 00000000 .....d...:.!.... - 1950 00000000 00003a08 21000000 00000000 ......:.!....... - 1960 00000000 42f62000 00000000 00000000 ....B. ......... - 1970 00080000 00000000 00000015 00000000 ................ - 1980 00000002 97081fa6 1900003a 15210000 ...........:.!.. - 1990 00000000 00000000 3a082100 00000000 ........:.!..... - 19a0 00000000 000043f6 20000000 00000000 ......C. ....... - 19b0 00000008 00000000 00000000 00150000 ................ - 19c0 00000000 00029808 1f3a1521 00000000 .........:.!.... - 19d0 00000000 00003a08 21000000 00000000 ......:.!....... - 19e0 00000000 0000002f 00000000 00000000 ......./........ - 19f0 b5260000 ff190000 23015502 7f000022 .&......#.U...." - 1a00 00000000 00000000 b5260000 23015502 .........&..#.U. - 1a10 7e000000 39552000 00000000 00000000 ~...9U ......... - 1a20 00000000 00000001 c005971a 00003a74 ..............:t - 1a30 20000000 00000000 0000003a 67200000 ..........:g .. - 1a40 00000000 00000000 44000000 00458120 ........D....E. - 1a50 00000391 907f419d 21000000 00000000 ......A.!....... - 1a60 00000001 00000000 00027808 053ac921 ..........x..:.! - 1a70 00000000 00000000 00003abc 21000000 ..........:.!... - 1a80 00000000 0000003a af210000 00000000 .......:.!...... - 1a90 00000000 00000046 a01f0000 00000000 .......F........ - 1aa0 00000000 02000000 000001d6 0c3ab21f .............:.. - 1ab0 00000000 00000000 00004400 0000003c ..........D....< - 1ac0 bf1f0000 00000000 00000000 418f2000 ............A. . - 1ad0 00000000 00000000 00010000 00000002 ................ - 1ae0 c4080c3a c8200000 00000000 00000000 ...:. .......... - 1af0 3abb2000 00000000 00000000 003aae20 :. ..........:. - 1b00 00000000 00000000 00003aa1 20000000 ..........:. ... - 1b10 00000000 0000003f 23210000 00000000 .......?#!...... - 1b20 00000000 04000000 000002de 070d4d1b ..............M. - 1b30 00003a42 21000000 00000000 0000003a ..:B!..........: - 1b40 35210000 00000000 00000000 0041d620 5!...........A. - 1b50 00000000 00000000 00000400 00000000 ................ - 1b60 02e50710 3ae82000 00000000 00000000 ....:. ......... - 1b70 00000000 00003700 00000001 a10e6100 ......7.......a. - 1b80 00000000 00000000 0000ad00 00000000 ................ - 1b90 0000019c ca1c0000 31000000 0001a131 ........1......1 - 1ba0 cb050000 00000000 00000000 31000000 ............1... - 1bb0 0001a13e 34000000 00000000 00000000 ...>4........... - 1bc0 31000000 0001a152 61000000 00000000 1......Ra....... - 1bd0 00000000 38000000 0001a320 1f060000 ....8...... .... - 1be0 02914c2e 00000000 00000000 7b000000 ..L.........{... - 1bf0 00000000 961c0000 36690001 a60e4c00 ........6i....L. - 1c00 00000000 00000000 00002f00 00000000 ........../..... - 1c10 00000001 1d00002d 1c000023 0155027c .......-...#.U.| - 1c20 00230154 02730023 01510138 002f0000 .#.T.s.#.Q.8./.. - 1c30 00000000 0000b526 0000451c 00002301 .......&..E...#. - 1c40 55027600 002f0000 00000000 0000011d U.v../.......... - 1c50 0000631c 00002301 55027c00 23015402 ..c...#.U.|.#.T. - 1c60 7600002f 00000000 00000000 b5260000 v../.........&.. - 1c70 7b1c0000 23015502 76000022 00000000 {...#.U.v..".... - 1c80 00000000 011d0000 23015502 7c002301 ........#.U.|.#. - 1c90 54027600 00002f00 00000000 00000094 T.v.../......... - 1ca0 1d0000b5 1c000023 0155027c 00230154 .......#.U.|.#.T - 1cb0 03f30151 00220000 00000000 0000ca1c ...Q.".......... - 1cc0 00002301 55027c00 00003700 00000001 ..#.U.|...7..... - 1cd0 9c0e6100 00000000 00000000 00000300 ..a............. - 1ce0 00000000 0000019c fb1c0000 47000000 ............G... - 1cf0 00019c4e fb1c0000 01550008 083a0600 ...N.....U...:.. - 1d00 00300000 00000192 06000000 00000000 .0.............. - 1d10 00270000 00000000 00019c8e 1d000047 .'.............G - 1d20 00000000 01923d8e 1d000001 55326b65 ......=.....U2ke - 1d30 79000192 55fb0200 00000000 00000000 y...U........... - 1d40 00310000 00000192 61340000 00000000 .1......a4...... - 1d50 00000000 00350000 00000194 12610000 .....5.......a.. - 1d60 00000000 00000000 00480000 00000000 .........H...... - 1d70 00002200 00000000 00003669 00019511 ..".......6i.... - 1d80 34000000 00000000 00000000 00000808 4............... - 1d90 1f060000 30000000 00018d06 00000000 ....0........... - 1da0 00000000 0f000000 00000000 019ccf1d ................ - 1db0 00004700 00000001 8d3b8e1d 00000155 ..G......;.....U - 1dc0 47000000 00018d54 61000000 01540037 G......Ta....T.7 - 1dd0 00000000 016e054c 00000000 00000000 .....n.L........ - 1de0 00000088 00000000 00000001 9c6a1e00 .............j.. - 1df0 00326100 016e1c19 03000000 00000000 .2a..n.......... - 1e00 00000032 6200016e 2b190300 00000000 ...2b..n+....... - 1e10 00000000 00366161 00017021 88070000 .....6aa..p!.... - 1e20 00000000 00000000 36626200 01712188 ........6bb..q!. - 1e30 07000000 00000000 00000035 00000000 ...........5.... - 1e40 0172094c 00000000 00000000 0000002c .r.L..........., - 1e50 00000000 00000000 c2260000 49000000 .........&..I... - 1e60 00000000 00c22600 00004a00 00000001 ......&...J..... - 1e70 5f0601aa 1e00004b 00000000 015f2fcb _......K....._/. - 1e80 0500004b 00000000 015f3c34 0000004c ...K....._<4...L - 1e90 4d690001 640e4c00 00004c4d 74616700 Mi..d.L...LMtag. - 1ea0 01651fcb 05000000 00004a00 00000001 .e........J..... - 1eb0 4a0601d0 1e00004b 00000000 014a2acb J......K.....J*. - 1ec0 0500004e 73726300 014a4c88 07000000 ...Nsrc..JL..... - 1ed0 37000000 00011c05 4c000000 00000000 7.......L....... - 1ee0 00000000 c8000000 00000000 019c6f1f ..............o. - 1ef0 00003261 00011c31 b2060000 00000000 ..2a...1........ - 1f00 00000000 32620001 1c4fb206 00000000 ....2b...O...... - 1f10 00000000 00004400 00000036 69000125 ......D....6i..% - 1f20 11340000 00000000 00000000 00440000 .4...........D.. - 1f30 00004f00 00000001 26258807 00003500 ..O.....&%....5. - 1f40 00000001 27258807 00000000 00000000 ....'%.......... - 1f50 00002c00 00000000 000000c2 2600002c ..,.........&.., - 1f60 00000000 00000000 c2260000 00000037 .........&.....7 - 1f70 00000000 01170e61 00000000 00000000 .......a........ - 1f80 00000004 00000000 00000001 9ca01f00 ................ - 1f90 00470000 00000117 45b20600 00015500 .G......E.....U. - 1fa0 50000000 0002b508 1d660300 0003cd1f P........f...... - 1fb0 00002500 00000002 b5083fcd 1f000026 ..%.......?....& - 1fc0 68333200 02b7080d 08040000 00080880 h32............. - 1fd0 03000050 00000000 027f0801 41030000 ...P........A... - 1fe0 03492000 00250000 0000027f 081d4920 .I ..%........I - 1ff0 00002500 00000002 7f083019 03000051 ..%.......0....Q - 2000 6c656e00 027f083e 34000000 4c267000 len....>4...L&p. - 2010 02860817 35050000 27000000 00028708 ....5...'....... - 2020 1d400500 00523820 00002670 33320002 [email protected] ..&p32.. - 2030 9408204f 20000000 4c270000 0000029f .. O ...L'...... - 2040 08214005 00000000 00080873 03000008 [email protected].... - 2050 08150400 00500000 0000026f 081e4103 .....P.....o..A. - 2060 0000038f 20000025 00000000 026f0839 .... ..%.....o.9 - 2070 49200000 25000000 00026f08 50660300 I ..%.....o.Pf.. - 2080 00270000 00000271 08137303 00000050 .'.....q..s....P - 2090 00000000 02cb0701 08040000 01d62000 .............. . - 20a0 00516833 320002cb 07180804 00005170 .Qh32.........Qp - 20b0 74720002 cb072b35 05000051 6c656e00 tr....+5...Qlen. - 20c0 02cb0737 34000000 25000000 0002cb07 ...74...%....... - 20d0 4a360400 00005000 00000002 b0071008 J6....P......... - 20e0 04000001 f6200000 51683332 0002b007 ..... ..Qh32.... - 20f0 28080400 00005000 00000002 7a071008 (.....P.....z... - 2100 04000001 23210000 51616363 00027a07 ....#!..Qacc..z. - 2110 24080400 00250000 0000027a 07310804 $....%.....z.1.. - 2120 00000050 00000000 02460701 08040000 ...P.....F...... - 2130 03502100 00517074 72000246 07201903 .P!..Qptr..F. .. - 2140 00002500 00000002 46073336 04000000 ..%.....F.36.... - 2150 50000000 00023a07 1a080400 00037021 P.....:.......p! - 2160 00005170 74720002 3a073319 03000000 ..Qptr..:.3..... - 2170 50000000 00029806 10080400 00019d21 P..............! - 2180 00002500 00000002 98062719 03000026 ..%.......'....& - 2190 76616c00 029a060d 08040000 00500000 val..........P.. - 21a0 000002c8 050e6800 000001d7 21000025 ......h.....!..% - 21b0 00000000 02c8051f 68000000 51737263 ........h...Qsrc - 21c0 0002c805 31190300 00250000 000002c8 ....1....%...... - 21d0 053d3400 00000053 aa1e0000 00000000 .=4....S........ - 21e0 00000000 6f000000 00000000 019c6822 ....o.........h" - 21f0 00003ab7 1e000000 00000000 0000003a ..:............: - 2200 c31e0000 00000000 00000000 54aa1e00 ............T... - 2210 00000000 00000000 00010000 00000000 ................ - 2220 0000000d 00000000 00000001 4a065a22 ............J.Z" - 2230 00003ac3 1e000000 00000000 0000003a ..:............: - 2240 b71e0000 00000000 00000000 2c000000 ............,... - 2250 00000000 00ce2600 00002c00 00000000 ......&...,..... - 2260 000000ce 26000000 536a1e00 00000000 ....&...Sj...... - 2270 00000000 00610000 00000000 00019c22 .....a........." - 2280 2300003a 771e0000 00000000 00000000 #..:w........... - 2290 3a831e00 00000000 00000000 00556a1e :............Uj. - 22a0 00000000 0000015f 0640831e 00004077 ......._.@....@w - 22b0 1e00003d 8f1e0000 00000000 0b230000 ...=.........#.. - 22c0 3c901e00 00000000 00000000 00569a1e <............V.. - 22d0 00000000 00000000 00001b00 00000000 ................ - 22e0 00003c9b 1e000000 00000000 0000002c ..<............, - 22f0 00000000 00000000 90260000 2c000000 .........&..,... - 2300 00000000 00902600 00000029 00000000 ......&....).... - 2310 00000000 90260000 23015503 f3015500 .....&..#.U...U. - 2320 000053b8 06000000 00000000 0000003d ..S............= - 2330 03000000 00000001 9c902600 003ac606 ..........&..:.. - 2340 00000000 00000000 00003ad3 06000000 ..........:..... - 2350 00000000 0000003a e0060000 00000000 .......:........ - 2360 00000000 3aed0600 00000000 00000000 ....:........... - 2370 003efa06 00003e05 0700003e 12070000 .>....>....>.... - 2380 3e1f0700 003e2c07 00003e39 0700003e >....>,...>9...> - 2390 46070000 3e530700 003e6007 00003e6d F...>S...>`...>m - 23a0 0700003e 7a070000 3fb80600 00000000 ...>z...?....... - 23b0 00000000 00010000 00000001 32010674 ............2..t - 23c0 2600003a ed060000 00000000 00000000 &..:............ - 23d0 3ae00600 00000000 00000000 003ad306 :............:.. - 23e0 00000000 00000000 00003ac6 06000000 ..........:..... - 23f0 00000000 00000044 00000000 3cfa0600 .......D....<... - 2400 00000000 00000000 003c0507 00000000 .........<...... - 2410 00000000 00003c12 07000000 00000000 ......<......... - 2420 0000003c 1f070000 00000000 00000000 ...<............ - 2430 3c2c0700 00000000 00000000 003c3907 <,...........<9. - 2440 00000000 00000000 00003c46 07000000 ..........<F.... - 2450 00000000 0000003c 53070000 00000000 .......<S....... - 2460 00000000 3c600700 00000000 00000000 ....<`.......... - 2470 003c6d07 00000000 00000000 00003c7a .<m...........<z - 2480 07000000 00000000 0000002f 00000000 .........../.... - 2490 00000000 9d260000 a4240000 23015503 .....&...$..#.U. - 24a0 0a000400 2f000000 00000000 009d2600 ..../.........&. - 24b0 00bd2400 00230155 030a0004 002f0000 ..$..#.U...../.. - 24c0 00000000 0000da26 0000d524 00002301 .......&...$..#. - 24d0 54027e00 002f0000 00000000 0000aa26 T.~../.........& - 24e0 0000ff24 00002301 55087c00 7e00227f ...$..#.U.|.~.". - 24f0 001c2301 54027d00 23015102 7600002c ..#.T.}.#.Q.v.., - 2500 00000000 00000000 9d260000 2f000000 .........&../... - 2510 00000000 00aa2600 002a2500 00230154 ......&..*%..#.T - 2520 027c0023 01510276 00002f00 00000000 .|.#.Q.v../..... - 2530 00000090 26000042 25000023 0155027c ....&..B%..#.U.| - 2540 00004900 00000000 00000090 2600002f ..I.........&../ - 2550 00000000 00000000 b5260000 69250000 .........&..i%.. - 2560 23015504 91987f06 002f0000 00000000 #.U....../...... - 2570 0000b526 00008325 00002301 550491a0 ...&...%..#.U... - 2580 7f06002f 00000000 00000000 da260000 .../.........&.. - 2590 9b250000 23015402 7300002f 00000000 .%..#.T.s../.... - 25a0 00000000 aa260000 c1250000 23015502 .....&...%..#.U. - 25b0 7d002301 54049198 7f062301 51027f00 }.#.T.....#.Q... - 25c0 002f0000 00000000 0000aa26 0000ec25 ./.........&...% - 25d0 00002301 55057d00 7f002223 01540491 ..#.U.}..."#.T.. - 25e0 a07f0623 01510491 a87f0600 2f000000 ...#.Q....../... - 25f0 00000000 00b52600 00062600 00230155 ......&...&..#.U - 2600 0491987f 06002f00 00000000 000000da ....../......... - 2610 2600001e 26000023 01540273 00002f00 &...&..#.T.s../. - 2620 00000000 000000aa 26000044 26000023 ........&..D&..# - 2630 0155027d 00230154 0491987f 06230151 .U.}.#.T.....#.Q - 2640 027f0000 2f000000 00000000 00b52600 ..../.........&. - 2650 005e2600 00230155 0491987f 06002200 .^&..#.U......". - 2660 00000000 000000da 26000023 01540273 ........&..#.T.s - 2670 00000000 22000000 00000000 00ce2600 ....".........&. - 2680 00230155 09030000 00000000 00000000 .#.U............ - 2690 57000000 00000000 000b3302 0d570000 W.........3..W.. - 26a0 00000000 00000b1b 020e5800 00000000 ..........X..... - 26b0 0000000d 00570000 00000000 00000c81 .....W.......... - 26c0 010f5900 00000000 0000000c 890c5900 ..Y...........Y. - 26d0 00000000 0000000c a70e5700 00000000 ..........W..... - 26e0 0000000b 25020e00 ....%... -Contents of section .debug_abbrev: - 0000 01110125 0e130b03 0e1b0e11 01120710 ...%............ - 0010 17000002 24000b0b 3e0b030e 00000316 ....$...>....... - 0020 00030e3a 0b3b0b39 0b491300 00042600 ...:.;.9.I....&. - 0030 49130000 0524000b 0b3e0b03 08000006 I....$...>...... - 0040 0f000b0b 00000737 00491300 00080f00 .......7.I...... - 0050 0b0b4913 00000913 01030e0b 0b3a0b3b ..I..........:.; - 0060 0b390b01 1300000a 0d00030e 3a0b3b0b .9..........:.;. - 0070 390b4913 380b0000 0b160003 0e3a0b3b 9.I.8........:.; - 0080 0b390b00 000c1300 030e3c19 00000d01 .9........<..... - 0090 01491301 1300000e 21004913 2f0b0000 .I......!.I./... - 00a0 0f340003 0e3a0b3b 0b390b49 133f193c .4...:.;.9.I.?.< - 00b0 19000010 21000000 11260000 00120401 ....!....&...... - 00c0 3e0b0b0b 49133a0b 3b05390b 01130000 >...I.:.;.9..... - 00d0 13280003 0e1c0b00 00141600 030e3a0b .(............:. - 00e0 3b05390b 49130000 15130103 0e0b0b3a ;.9.I..........: - 00f0 0b3b0539 0b011300 00160d00 030e3a0b .;.9..........:. - 0100 3b05390b 4913380b 0000170d 0003083a ;.9.I.8........: - 0110 0b3b0539 0b491338 0b000018 3400030e .;.9.I.8....4... - 0120 3a0b3b05 390b4913 88010b1c 0a000019 :.;.9.I......... - 0130 0401030e 3e0b0b0b 49133a0b 3b0b390b ....>...I.:.;.9. - 0140 01130000 1a17010b 0b3a0b3b 0b390b01 .........:.;.9.. - 0150 1300001b 0d00030e 3a0b3b0b 390b4913 ........:.;.9.I. - 0160 00001c0d 0003083a 0b3b0b39 0b491338 .......:.;.9.I.8 - 0170 0b00001d 0d004913 380b0000 1e2e013f ......I.8......? - 0180 19030e3a 0b3b0539 0b271949 13110112 ...:.;.9.'.I.... - 0190 07401897 42190113 00001f05 0003083a [email protected]..........: - 01a0 0b3b0539 0b491302 17b74217 00002034 .;.9.I....B... 4 - 01b0 0003083a 0b3b0539 0b491302 18000021 ...:.;.9.I.....! - 01c0 3400030e 3a0b3b05 390b4913 02180000 4...:.;.9.I..... - 01d0 22898201 01110131 13000023 8a820100 "......1...#.... - 01e0 02189142 18000024 2e013f19 030e3a0b ...B...$..?...:. - 01f0 3b05390b 2719200b 01130000 25050003 ;.9.'. .....%... - 0200 0e3a0b3b 05390b49 13000026 34000308 .:.;.9.I...&4... - 0210 3a0b3b05 390b4913 00002734 00030e3a :.;.9.I...'4...: - 0220 0b3b0539 0b491300 00280500 030e3a0b .;.9.I...(....:. - 0230 3b05390b 49130217 b7421700 00298982 ;.9.I....B...).. - 0240 01011101 95421931 1300002a 2e013f19 .....B.1...*..?. - 0250 030e3a0b 3b05390b 27191101 12074018 ..:.;.9.'.....@. - 0260 97421901 1300002b 89820101 11019542 .B.....+.......B - 0270 19311301 1300002c 89820100 11013113 .1.....,......1. - 0280 00002d34 0003083a 0b3b0539 0b491302 ..-4...:.;.9.I.. - 0290 17b74217 00002e0b 01110112 07011300 ..B............. - 02a0 002f8982 01011101 31130113 0000302e ./......1.....0. - 02b0 013f1903 0e3a0b3b 0b390b27 19110112 .?...:.;.9.'.... - 02c0 07401897 42190113 00003105 00030e3a [email protected]....: - 02d0 0b3b0b39 0b491302 17b74217 00003205 .;.9.I....B...2. - 02e0 0003083a 0b3b0b39 0b491302 17b74217 ...:.;.9.I....B. - 02f0 00003334 00030e3a 0b3b0539 0b491302 ..34...:.;.9.I.. - 0300 17b74217 0000340b 01551701 13000035 ..B...4..U.....5 - 0310 3400030e 3a0b3b0b 390b4913 0217b742 4...:.;.9.I....B - 0320 17000036 34000308 3a0b3b0b 390b4913 ...64...:.;.9.I. - 0330 0217b742 17000037 2e013f19 030e3a0b ...B...7..?...:. - 0340 3b0b390b 27194913 11011207 40189742 ;.9.'[email protected] - 0350 19011300 00383400 030e3a0b 3b0b390b .....84...:.;.9. - 0360 49130218 0000391d 01311352 01b84205 I.....9..1.R..B. - 0370 5517580b 590b570b 01130000 3a050031 U.X.Y.W.....:..1 - 0380 130217b7 42170000 3b0b0131 13551700 ....B...;..1.U.. - 0390 003c3400 31130217 b7421700 003d0b01 .<4.1....B...=.. - 03a0 31135517 01130000 3e340031 1300003f 1.U.....>4.1...? - 03b0 1d013113 5201b842 05551758 0b590557 ..1.R..B.U.X.Y.W - 03c0 0b011300 00400500 31130000 411d0131 [email protected] - 03d0 135201b8 42055517 580b5905 570b0000 .R..B.U.X.Y.W... - 03e0 421d0131 135201b8 42051101 1207580b B..1.R..B.....X. - 03f0 5905570b 01130000 431d0131 135201b8 Y.W.....C..1.R.. - 0400 42051101 1207580b 5905570b 0000440b B.....X.Y.W...D. - 0410 01551700 00453400 31130218 0000461d .U...E4.1.....F. - 0420 01311352 01b84205 5517580b 590b570b .1.R..B.U.X.Y.W. - 0430 00004705 00030e3a 0b3b0b39 0b491302 ..G....:.;.9.I.. - 0440 18000048 0b011101 12070000 49898201 ...H........I... - 0450 00110195 42193113 00004a2e 013f1903 ....B.1...J..?.. - 0460 0e3a0b3b 0b390b27 19200b01 1300004b .:.;.9.'. .....K - 0470 0500030e 3a0b3b0b 390b4913 00004c0b ....:.;.9.I...L. - 0480 0100004d 34000308 3a0b3b0b 390b4913 ...M4...:.;.9.I. - 0490 00004e05 0003083a 0b3b0b39 0b491300 ..N....:.;.9.I.. - 04a0 004f3400 030e3a0b 3b0b390b 49130000 .O4...:.;.9.I... - 04b0 502e0103 0e3a0b3b 05390b27 19491320 P....:.;.9.'.I. - 04c0 0b011300 00510500 03083a0b 3b05390b .....Q....:.;.9. - 04d0 49130000 520b0101 13000053 2e013113 I...R......S..1. - 04e0 11011207 40189742 19011300 00541d01 [email protected].. - 04f0 31135201 b8420511 01120758 0b590b57 1.R..B.....X.Y.W - 0500 0b011300 00551d01 31135517 580b590b .....U..1.U.X.Y. - 0510 570b0000 560b0131 13110112 07000057 W...V..1.......W - 0520 2e003f19 3c196e0e 030e3a0b 3b05390b ..?.<.n...:.;.9. - 0530 0000582e 003f193c 196e0e03 0e3a0b3b ..X..?.<.n...:.; - 0540 0b000059 2e003f19 3c196e0e 030e3a0b ...Y..?.<.n...:. - 0550 3b0b390b 000000 ;.9.... -Contents of section .debug_loc: - 0000 00000000 40130000 00000000 4b130000 [email protected]... - 0010 00000000 0100554b 13000000 00000061 ......UK.......a - 0020 13000000 00000004 00f30155 9f000000 ...........U.... - 0030 00000000 00000000 00000000 00000000 ................ - 0040 00f00f00 00000000 00f90f00 00000000 ................ - 0050 00010055 f90f0000 00000000 fe0f0000 ...U............ - 0060 00000000 0400f301 559f0000 00000000 ........U....... - 0070 00000000 00000000 00000000 00000000 ................ - 0080 f00f0000 00000000 f60f0000 00000000 ................ - 0090 010054f6 0f000000 000000fd 0f000000 ..T............. - 00a0 00000001 0051fd0f 00000000 0000fe0f .....Q.......... - 00b0 00000000 00000400 f301549f 00000000 ..........T..... - 00c0 00000000 00000000 00000000 00000000 ................ - 00d0 00000000 00000000 0000c00f 00000000 ................ - 00e0 0000cd0f 00000000 00000100 55cd0f00 ............U... - 00f0 00000000 00ce0f00 00000000 00010053 ...............S - 0100 ce0f0000 00000000 d20f0000 00000000 ................ - 0110 010055d2 0f000000 000000d3 0f000000 ..U............. - 0120 00000004 00f30155 9fd30f00 00000000 .......U........ - 0130 00e80f00 00000000 00010053 e80f0000 ...........S.... - 0140 00000000 ec0f0000 00000000 010055ec ..............U. - 0150 0f000000 000000ed 0f000000 00000004 ................ - 0160 00f30155 9f000000 00000000 00000000 ...U............ - 0170 00000000 00000000 00000000 00000030 ...............0 - 0180 0f000000 0000003b 0f000000 00000001 .......;........ - 0190 00553b0f 00000000 0000650f 00000000 .U;.......e..... - 01a0 00000100 56650f00 00000000 00700f00 ....Ve.......p.. - 01b0 00000000 000400f3 01559f70 0f000000 .........U.p.... - 01c0 000000b3 0f000000 00000001 0056b30f .............V.. - 01d0 00000000 0000b60f 00000000 00000400 ................ - 01e0 f301559f 00000000 00000000 00000000 ..U............. - 01f0 00000000 00000000 00000000 0000470f ..............G. - 0200 00000000 00004b0f 00000000 00000100 ......K......... - 0210 504b0f00 00000000 006f0f00 00000000 PK.......o...... - 0220 0001005c 6f0f0000 00000000 700f0000 ...\o.......p... - 0230 00000000 01005070 0f000000 000000b5 ......Pp........ - 0240 0f000000 00000001 005cb50f 00000000 .........\...... - 0250 0000b60f 00000000 00000100 50000000 ............P... - 0260 00000000 00000000 00000000 00020000 ................ - 0270 00830f00 00000000 00850f00 00000000 ................ - 0280 00020030 9f8c0f00 00000000 00980f00 ...0............ - 0290 00000000 00010053 00000000 00000000 .......S........ - 02a0 00000000 00000000 00000000 0000c00e ................ - 02b0 00000000 0000ec0e 00000000 00000100 ................ - 02c0 55ec0e00 00000000 00200f00 00000000 U........ ...... - 02d0 00010056 200f0000 00000000 250f0000 ...V .......%... - 02e0 00000000 0400f301 559f0000 00000000 ........U....... - 02f0 00000000 00000000 00000000 00000000 ................ - 0300 c00e0000 00000000 f70e0000 00000000 ................ - 0310 010054f7 0e000000 00000000 0f000000 ..T............. - 0320 00000001 0053000f 00000000 0000250f .....S........%. - 0330 00000000 00000400 f301549f 00000000 ..........T..... - 0340 00000000 00000000 00000000 00000000 ................ - 0350 c00e0000 00000000 f70e0000 00000000 ................ - 0360 010051f7 0e000000 00000025 0f000000 ..Q........%.... - 0370 00000004 00f30151 9f000000 00000000 .......Q........ - 0380 00000000 00000000 000000c4 0e000000 ................ - 0390 000000f7 0e000000 00000001 00500000 .............P.. - 03a0 00000000 00000000 00000000 00000000 ................ - 03b0 0000fe0e 00000000 0000000f 00000000 ................ - 03c0 00000100 50000f00 00000000 00240f00 ....P........$.. - 03d0 00000000 0001005d 00000000 00000000 .......]........ - 03e0 00000000 00000000 0300fe0e 00000000 ................ - 03f0 0000000f 00000000 00000200 309f0000 ............0... - 0400 00000000 00000000 00000000 00000000 ................ - 0410 00000000 500e0000 00000000 770e0000 ....P.......w... - 0420 00000000 01005577 0e000000 000000b9 ......Uw........ - 0430 0e000000 00000001 0056b90e 00000000 .........V...... - 0440 0000be0e 00000000 00000400 f301559f ..............U. - 0450 00000000 00000000 00000000 00000000 ................ - 0460 00000000 00008a0e 00000000 00008a0e ................ - 0470 00000000 00000100 508a0e00 00000000 ........P....... - 0480 00bd0e00 00000000 0001005d bd0e0000 ...........].... - 0490 00000000 be0e0000 00000000 0300f301 ................ - 04a0 55000000 00000000 00000000 00000000 U............... - 04b0 00020000 008a0e00 00000000 008a0e00 ................ - 04c0 00000000 00020030 9f8a0e00 00000000 .......0........ - 04d0 009d0e00 00000000 0001005c 00000000 ...........\.... - 04e0 00000000 00000000 00000000 00000000 ................ - 04f0 0000000e 00000000 00000e0e 00000000 ................ - 0500 00000100 550e0e00 00000000 00260e00 ....U........&.. - 0510 00000000 00010056 260e0000 00000000 .......V&....... - 0520 440e0000 00000000 0400f301 559f0000 D...........U... - 0530 00000000 00000000 00000000 00000000 ................ - 0540 00000000 000e0000 00000000 130e0000 ................ - 0550 00000000 01005413 0e000000 00000043 ......T........C - 0560 0e000000 00000001 005c430e 00000000 .........\C..... - 0570 0000440e 00000000 00000400 f301549f ..D...........T. - 0580 00000000 00000000 00000000 00000000 ................ - 0590 00000000 00001d0e 00000000 0000220e ..............". - 05a0 00000000 00000100 50220e00 00000000 ........P"...... - 05b0 00400e00 00000000 00010053 400e0000 [email protected]@... - 05c0 00000000 440e0000 00000000 01005000 ....D.........P. - 05d0 00000000 00000000 00000000 00000000 ................ - 05e0 000000f0 0d000000 000000f6 0d000000 ................ - 05f0 00000001 0055f60d 00000000 0000f70d .....U.......... - 0600 00000000 00000400 f301559f 00000000 ..........U..... - 0610 00000000 00000000 00000000 00000000 ................ - 0620 f00d0000 00000000 f60d0000 00000000 ................ - 0630 010054f6 0d000000 000000f7 0d000000 ..T............. - 0640 00000004 00f30154 9f000000 00000000 .......T........ - 0650 00000000 00000000 00000000 00000050 ...............P - 0660 03000000 000000ed 03000000 00000001 ................ - 0670 0055ed03 00000000 0000ba0d 00000000 .U.............. - 0680 00000400 f301559f ba0d0000 00000000 ......U......... - 0690 ea0d0000 00000000 01005500 00000000 ..........U..... - 06a0 00000000 00000000 00000000 00000000 ................ - 06b0 00000050 03000000 000000ed 03000000 ...P............ - 06c0 00000001 0054ed03 00000000 0000ba0d .....T.......... - 06d0 00000000 00000400 f301549f ba0d0000 ..........T..... - 06e0 00000000 c40d0000 00000000 010054c4 ..............T. - 06f0 0d000000 000000ea 0d000000 00000004 ................ - 0700 00f30154 9f000000 00000000 00000000 ...T............ - 0710 00000000 00000000 00000000 00500300 .............P.. - 0720 00000000 00890300 00000000 00010051 ...............Q - 0730 89030000 00000000 ed030000 00000000 ................ - 0740 030091a0 7fed0300 00000000 00ba0d00 ................ - 0750 00000000 000400f3 01519fba 0d000000 .........Q...... - 0760 000000ea 0d000000 00000003 0091a07f ................ - 0770 00000000 00000000 00000000 00000000 ................ - 0780 02000000 c0030000 00000000 ed030000 ................ - 0790 00000000 0200309f ba0d0000 00000000 ......0......... - 07a0 ea0d0000 00000000 0200309f 00000000 ..........0..... - 07b0 00000000 00000000 00000000 00000000 ................ - 07c0 00010000 07040000 00000000 0a040000 ................ - 07d0 00000000 0100500a 04000000 00000014 ......P......... - 07e0 05000000 00000001 005e9107 00000000 .........^...... - 07f0 0000bc07 00000000 00000100 5e3a0a00 ............^:.. - 0800 00000000 00e60a00 00000000 0001005e ...............^ - 0810 00000000 00000000 00000000 00000000 ................ - 0820 00000001 00000000 07040000 00000000 ................ - 0830 14050000 00000000 01005f91 07000000 .........._..... - 0840 000000bc 07000000 00000001 005f3a0a ............._:. - 0850 00000000 0000450a 00000000 00000100 ......E......... - 0860 54450a00 00000000 00040b00 00000000 TE.............. - 0870 0001005f 00000000 00000000 00000000 ..._............ - 0880 00000000 00000001 00000704 00000000 ................ - 0890 00001405 00000000 00000300 91c87e91 ..............~. - 08a0 07000000 000000bc 07000000 00000003 ................ - 08b0 0091c87e 3a0a0000 00000000 040b0000 ...~:........... - 08c0 00000000 030091c8 7e000000 00000000 ........~....... - 08d0 00000000 00000000 00030000 00000000 ................ - 08e0 03030000 00000100 00000101 00070400 ................ - 08f0 00000000 003e0400 00000000 0001005f .....>........._ - 0900 3e040000 00000000 65040000 00000000 >.......e....... - 0910 01005465 04000000 00000072 04000000 ..Te.......r.... - 0920 00000001 00527204 00000000 00008704 .....Rr......... - 0930 00000000 00000300 72709fbb 04000000 ........rp...... - 0940 00000008 05000000 00000001 00520805 .............R.. - 0950 00000000 00001405 00000000 00000100 ................ - 0960 54910700 00000000 00bc0700 00000000 T............... - 0970 0001005f 3a0a0000 00000000 450a0000 ..._:.......E... - 0980 00000000 01005445 0a000000 000000d4 ......TE........ - 0990 0a000000 00000001 005fd40a 00000000 ........._...... - 09a0 0000f90a 00000000 00000100 54000000 ............T... - 09b0 00000000 00000000 00000000 00040000 ................ - 09c0 00000100 00000007 04000000 0000000a ................ - 09d0 04000000 00000006 007f0070 00229f0a ...........p.".. - 09e0 04000000 00000014 05000000 00000006 ................ - 09f0 007f007e 00229f91 07000000 000000bc ...~.".......... - 0a00 07000000 00000006 007f007e 00229f3a ...........~.".: - 0a10 0a000000 00000045 0a000000 00000006 .......E........ - 0a20 0074007e 00229f45 0a000000 000000e6 .t.~.".E........ - 0a30 0a000000 00000006 007f007e 00229f00 ...........~.".. - 0a40 00000000 00000000 00000000 00000008 ................ - 0a50 00000165 04000000 00000072 04000000 ...e.......r.... - 0a60 00000002 00720072 04000000 00000087 .....r.r........ - 0a70 04000000 00000002 00727000 00000000 .........rp..... - 0a80 00000000 00000000 00000008 01650400 .............e.. - 0a90 00000000 00870400 00000000 00010058 ...............X - 0aa0 00000000 00000000 00000000 00000000 ................ - 0ab0 0a018704 00000000 00009c04 00000000 ................ - 0ac0 00000100 5a000000 00000000 00000000 ....Z........... - 0ad0 00000000 000a019c 04000000 000000ae ................ - 0ae0 04000000 00000001 00550000 00000000 .........U...... - 0af0 00000000 00000000 00000a00 00000001 ................ - 0b00 ae040000 00000000 b1040000 00000000 ................ - 0b10 01005bb1 04000000 000000bb 04000000 ..[............. - 0b20 00000001 0051bb04 00000000 0000bb04 .....Q.......... - 0b30 00000000 00000100 5b000000 00000000 ........[....... - 0b40 00000000 00000000 00010091 07000000 ................ - 0b50 000000ad 07000000 00000001 005e0000 .............^.. - 0b60 00000000 00000000 00000000 00000100 ................ - 0b70 91070000 00000000 ad070000 00000000 ................ - 0b80 01005f00 00000000 00000000 00000000 .._............. - 0b90 00000001 00000091 07000000 000000ac ................ - 0ba0 07000000 00000009 00910070 00220888 ...........p.".. - 0bb0 1c9fac07 00000000 0000ad07 00000000 ................ - 0bc0 00000f00 7d000cff ffffff1a 91002208 ....}.........". - 0bd0 881c9f00 00000000 00000000 00000000 ................ - 0be0 00000002 00000000 003a0a00 00000000 .........:...... - 0bf0 00540a00 00000000 000b0040 7d001c0c .T.........@}... - 0c00 ffffffff 1a9f540a 00000000 00005b0a ......T.......[. - 0c10 00000000 00000100 515b0a00 00000000 ........Q[...... - 0c20 005c0a00 00000000 000b0040 7d001c0c .\.........@}... - 0c30 ffffffff 1a9f0000 00000000 00000000 ................ - 0c40 00000000 00000200 00003a0a 00000000 ..........:..... - 0c50 0000450a 00000000 00000100 54450a00 ..E.........TE.. - 0c60 00000000 005c0a00 00000000 0001005f .....\........._ - 0c70 00000000 00000000 00000000 00000000 ................ - 0c80 02000000 3a0a0000 00000000 5b0a0000 ....:.......[... - 0c90 00000000 09009100 70002208 881c9f5b ........p."....[ - 0ca0 0a000000 0000005c 0a000000 0000000f .......\........ - 0cb0 007d000c ffffffff 1a910022 08881c9f .}.........".... - 0cc0 00000000 00000000 00000000 00000000 ................ - 0cd0 02010101 01010100 5c0a0000 00000000 ........\....... - 0ce0 890a0000 00000000 040091f8 7e9f890a ............~... - 0cf0 00000000 0000a20a 00000000 00000400 ................ - 0d00 91fc7e9f a20a0000 00000000 bb0a0000 ..~............. - 0d10 00000000 04009180 7f9fbb0a 00000000 ................ - 0d20 0000d90a 00000000 00000400 91847f9f ................ - 0d30 00000000 00000000 00000000 00000000 ................ - 0d40 09025c0a 00000000 00006e0a 00000000 ..\.......n..... - 0d50 00000300 91f87e00 00000000 00000000 ......~......... - 0d60 00000000 00000009 00000101 025c0a00 .............\.. - 0d70 00000000 006b0a00 00000000 00030091 .....k.......... - 0d80 e87e6b0a 00000000 00006e0a 00000000 .~k.......n..... - 0d90 00000100 506e0a00 00000000 006e0a00 ....Pn.......n.. - 0da0 00000000 000a0070 0011b1f3 ddf1791e .......p......y. - 0db0 9f000000 00000000 00000000 00000000 ................ - 0dc0 00080289 0a000000 00000098 0a000000 ................ - 0dd0 00000003 0091fc7e 00000000 00000000 .......~........ - 0de0 00000000 00000000 08000001 0102890a ................ - 0df0 00000000 0000950a 00000000 00000300 ................ - 0e00 91ec7e95 0a000000 00000098 0a000000 ..~............. - 0e10 00000001 0050980a 00000000 0000980a .....P.......... - 0e20 00000000 00000a00 700011b1 f3ddf179 ........p......y - 0e30 1e9f0000 00000000 00000000 00000000 ................ - 0e40 00000802 a20a0000 00000000 b10a0000 ................ - 0e50 00000000 03009180 7f000000 00000000 ................ - 0e60 00000000 00000000 00080000 010102a2 ................ - 0e70 0a000000 000000ae 0a000000 00000003 ................ - 0e80 0091f07e ae0a0000 00000000 b10a0000 ...~............ - 0e90 00000000 010050b1 0a000000 000000b1 ......P......... - 0ea0 0a000000 0000000a 00700011 b1f3ddf1 .........p...... - 0eb0 791e9f00 00000000 00000000 00000000 y............... - 0ec0 00000008 02bb0a00 00000000 00ca0a00 ................ - 0ed0 00000000 00030091 847f0000 00000000 ................ - 0ee0 00000000 00000000 00000800 00010102 ................ - 0ef0 bb0a0000 00000000 c70a0000 00000000 ................ - 0f00 030091f4 7ec70a00 00000000 00ca0a00 ....~........... - 0f10 00000000 00010050 ca0a0000 00000000 .......P........ - 0f20 ca0a0000 00000000 0a007000 11b1f3dd ..........p..... - 0f30 f1791e9f 00000000 00000000 00000000 .y.............. - 0f40 00000000 00000000 e60a0000 00000000 ................ - 0f50 eb0a0000 00000000 010050eb 0a000000 ..........P..... - 0f60 000000fa 0a000000 00000001 005e0000 .............^.. - 0f70 00000000 00000000 00000000 00000000 ................ - 0f80 e60a0000 00000000 f90a0000 00000000 ................ - 0f90 01005400 00000000 00000000 00000000 ..T............. - 0fa0 00000000 00e60a00 00000000 00fa0a00 ................ - 0fb0 00000000 00040091 f87e9f00 00000000 .........~...... - 0fc0 00000000 00000000 00000000 00000000 ................ - 0fd0 00000000 00440500 00000000 00520500 .....D.......R.. - 0fe0 00000000 00010050 52050000 00000000 .......PR....... - 0ff0 89050000 00000000 01005ac4 0c000000 ..........Z..... - 1000 000000e3 0c000000 00000001 005a7e0d .............Z~. - 1010 00000000 0000990d 00000000 00000100 ................ - 1020 5a990d00 00000000 00ba0d00 00000000 Z............... - 1030 00030091 d87e0000 00000000 00000000 .....~.......... - 1040 00000000 00000000 00000000 44050000 ............D... - 1050 00000000 89050000 00000000 01005ec4 ..............^. - 1060 0c000000 00000010 0d000000 00000001 ................ - 1070 005e7e0d 00000000 0000ba0d 00000000 .^~............. - 1080 00000100 5e000000 00000000 00000000 ....^........... - 1090 00000000 00000000 00000044 05000000 ...........D.... - 10a0 00000054 06000000 00000003 0091c87e ...T...........~ - 10b0 980c0000 00000000 690d0000 00000000 ........i....... - 10c0 030091c8 7e7e0d00 00000000 00ba0d00 ....~~.......... - 10d0 00000000 00030091 c87e0000 00000000 .........~...... - 10e0 00000000 00000000 00000300 00000003 ................ - 10f0 03000000 00000100 00004405 00000000 ..........D..... - 1100 0000aa05 00000000 00000100 5eaa0500 ............^... - 1110 00000000 00ba0500 00000000 00010050 ...............P - 1120 ba050000 00000000 cc050000 00000000 ................ - 1130 03007070 9ffd0500 00000000 00480600 ..pp.........H.. - 1140 00000000 00010050 48060000 00000000 .......PH....... - 1150 54060000 00000000 01005e98 0c000000 T.........^..... - 1160 00000010 0d000000 00000001 005e640d .............^d. - 1170 00000000 0000690d 00000000 00000100 ......i......... - 1180 5e7e0d00 00000000 00ba0d00 00000000 ^~.............. - 1190 0001005e 00000000 00000000 00000000 ...^............ - 11a0 00000000 00000000 00000000 48050000 ............H... - 11b0 00000000 54060000 00000000 01005f98 ....T........._. - 11c0 0c000000 000000a8 0c000000 00000001 ................ - 11d0 005fc40c 00000000 0000690d 00000000 ._........i..... - 11e0 00000100 5f7e0d00 00000000 00ba0d00 ...._~.......... - 11f0 00000000 0001005f 00000000 00000000 ......._........ - 1200 00000000 00000000 08000001 aa050000 ................ - 1210 00000000 ba050000 00000000 02007000 ..............p. - 1220 ba050000 00000000 cc050000 00000000 ................ - 1230 02007070 00000000 00000000 00000000 ..pp............ - 1240 00000000 0801aa05 00000000 0000cc05 ................ - 1250 00000000 00000100 51000000 00000000 ........Q....... - 1260 00000000 00000000 000a01cc 05000000 ................ - 1270 000000de 05000000 00000001 00550000 .............U.. - 1280 00000000 00000000 00000000 00000a01 ................ - 1290 de050000 00000000 f0050000 00000000 ................ - 12a0 01005400 00000000 00000000 00000000 ..T............. - 12b0 0000000a 00000000 01f00500 00000000 ................ - 12c0 00f30500 00000000 0001005b f3050000 ...........[.... - 12d0 00000000 fd050000 00000000 010052fd ..............R. - 12e0 05000000 000000fd 05000000 00000001 ................ - 12f0 005b0000 00000000 00000000 00000000 .[.............. - 1300 00000000 a80c0000 00000000 ba0c0000 ................ - 1310 00000000 01005f00 00000000 00000000 ......_......... - 1320 00000000 00000000 00a80c00 00000000 ................ - 1330 00ba0c00 00000000 0001005e 00000000 ...........^.... - 1340 00000000 00000000 00000000 0000a80c ................ - 1350 00000000 0000ba0c 00000000 00000400 ................ - 1360 91f87e9f 00000000 00000000 00000000 ..~............. - 1370 00000000 01000000 0000c40c 00000000 ................ - 1380 0000dc0c 00000000 00000b00 407d001c ............@}.. - 1390 0cffffff ff1a9fdc 0c000000 000000e3 ................ - 13a0 0c000000 00000001 0051e30c 00000000 .........Q...... - 13b0 0000e40c 00000000 00000b00 407d001c ............@}.. - 13c0 0cffffff ff1a9f00 00000000 00000000 ................ - 13d0 00000000 00000001 00c40c00 00000000 ................ - 13e0 00e40c00 00000000 0001005e 00000000 ...........^.... - 13f0 00000000 00000000 00000000 01000000 ................ - 1400 c40c0000 00000000 e30c0000 00000000 ................ - 1410 09009100 70002208 881c9fe3 0c000000 ....p."......... - 1420 000000e4 0c000000 0000000f 007d000c .............}.. - 1430 ffffffff 1a910022 08881c9f 00000000 ......."........ - 1440 00000000 00000000 00000000 02010101 ................ - 1450 01010100 e40c0000 00000000 190d0000 ................ - 1460 00000000 040091f8 7e9f190d 00000000 ........~....... - 1470 0000320d 00000000 00000400 91fc7e9f ..2...........~. - 1480 320d0000 00000000 4b0d0000 00000000 2.......K....... - 1490 04009180 7f9f4b0d 00000000 0000690d ......K.......i. - 14a0 00000000 00000400 91847f9f 00000000 ................ - 14b0 00000000 00000000 00000000 0902e40c ................ - 14c0 00000000 0000f60c 00000000 00000300 ................ - 14d0 91f87e00 00000000 00000000 00000000 ..~............. - 14e0 00000009 00000101 02e40c00 00000000 ................ - 14f0 00f00c00 00000000 00030091 e87ef00c .............~.. - 1500 00000000 0000f60c 00000000 00000100 ................ - 1510 50f60c00 00000000 00f60c00 00000000 P............... - 1520 000a0070 0011b1f3 ddf1791e 9f000000 ...p......y..... - 1530 00000000 00000000 00000000 00080219 ................ - 1540 0d000000 00000028 0d000000 00000003 .......(........ - 1550 0091fc7e 00000000 00000000 00000000 ...~............ - 1560 00000000 08000001 0102190d 00000000 ................ - 1570 0000250d 00000000 00000300 91ec7e25 ..%...........~% - 1580 0d000000 00000028 0d000000 00000001 .......(........ - 1590 0050280d 00000000 0000280d 00000000 .P(.......(..... - 15a0 00000a00 700011b1 f3ddf179 1e9f0000 ....p......y.... - 15b0 00000000 00000000 00000000 00000802 ................ - 15c0 320d0000 00000000 410d0000 00000000 2.......A....... - 15d0 03009180 7f000000 00000000 00000000 ................ - 15e0 00000000 00080000 01010232 0d000000 ...........2.... - 15f0 0000003e 0d000000 00000003 0091f07e ...>...........~ - 1600 3e0d0000 00000000 410d0000 00000000 >.......A....... - 1610 01005041 0d000000 00000041 0d000000 ..PA.......A.... - 1620 0000000a 00700011 b1f3ddf1 791e9f00 .....p......y... - 1630 00000000 00000000 00000000 00000008 ................ - 1640 024b0d00 00000000 005a0d00 00000000 .K.......Z...... - 1650 00030091 847f0000 00000000 00000000 ................ - 1660 00000000 00000800 00010102 4b0d0000 ............K... - 1670 00000000 570d0000 00000000 030091f4 ....W........... - 1680 7e570d00 00000000 005a0d00 00000000 ~W.......Z...... - 1690 00010050 5a0d0000 00000000 5a0d0000 ...PZ.......Z... - 16a0 00000000 0a007000 11b1f3dd f1791e9f ......p......y.. - 16b0 00000000 00000000 00000000 00000000 ................ - 16c0 02000000 7e0d0000 00000000 990d0000 ....~........... - 16d0 00000000 01005a99 0d000000 0000009a ......Z......... - 16e0 0d000000 00000003 0091d87e 00000000 ...........~.... - 16f0 00000000 00000000 00000000 02007e0d ..............~. - 1700 00000000 00009a0d 00000000 00000100 ................ - 1710 5e000000 00000000 00000000 00000000 ^............... - 1720 00020000 007e0d00 00000000 00990d00 .....~.......... - 1730 00000000 00090091 00700022 08881c9f .........p.".... - 1740 990d0000 00000000 9a0d0000 00000000 ................ - 1750 0f007d00 0cffffff ff1a9100 2208881c ..}........."... - 1760 9f000000 00000000 00000000 00000000 ................ - 1770 00010000 000000c5 07000000 000000ef ................ - 1780 08000000 00000002 00389ff5 0b000000 .........8...... - 1790 00000098 0c000000 00000002 00389f69 .............8.i - 17a0 0d000000 0000007e 0d000000 00000002 .......~........ - 17b0 00389f00 00000000 00000000 00000000 .8.............. - 17c0 00000001 00000000 00c50700 00000000 ................ - 17d0 00ef0800 00000000 00030073 089ff50b ...........s.... - 17e0 00000000 0000980c 00000000 00000300 ................ - 17f0 73089f69 0d000000 0000007e 0d000000 s..i.......~.... - 1800 00000003 0073089f 00000000 00000000 .....s.......... - 1810 00000000 00000000 01000000 0000c507 ................ - 1820 00000000 0000ef08 00000000 00000300 ................ - 1830 91c87ef5 0b000000 00000098 0c000000 ..~............. - 1840 00000003 0091c87e 690d0000 00000000 .......~i....... - 1850 7e0d0000 00000000 030091c8 7e000000 ~...........~... - 1860 00000000 00000000 00000000 00040000 ................ - 1870 00000000 03030000 00000101 000000c5 ................ - 1880 07000000 00000003 08000000 00000003 ................ - 1890 0073089f 03080000 00000000 20080000 .s.......... ... - 18a0 00000000 01005e20 08000000 0000002a ......^ .......* - 18b0 08000000 00000001 00502a08 00000000 .........P*..... - 18c0 00003c08 00000000 00000300 70709f6d ..<.........pp.m - 18d0 08000000 0000007f 08000000 00000001 ................ - 18e0 0050bf08 00000000 0000ef08 00000000 .P.............. - 18f0 00000100 5ef50b00 00000000 00930c00 ....^........... - 1900 00000000 00030073 089f930c 00000000 .......s........ - 1910 0000980c 00000000 00000100 5e690d00 ............^i.. - 1920 00000000 007e0d00 00000000 00030073 .....~.........s - 1930 089f0000 00000000 00000000 00000000 ................ - 1940 00000500 00000000 c5070000 00000000 ................ - 1950 ef080000 00000000 03007310 9ff50b00 ..........s..... - 1960 00000000 00980c00 00000000 00030073 ...............s - 1970 109f690d 00000000 00007e0d 00000000 ..i.......~..... - 1980 00000300 73109f00 00000000 00000000 ....s........... - 1990 00000000 00000008 00000120 08000000 ........... .... - 19a0 0000002a 08000000 00000002 0070002a ...*.........p.* - 19b0 08000000 0000003c 08000000 00000002 .......<........ - 19c0 00707000 00000000 00000000 00000000 .pp............. - 19d0 00000008 01200800 00000000 003c0800 ..... .......<.. - 19e0 00000000 00010054 00000000 00000000 .......T........ - 19f0 00000000 00000000 0a013c08 00000000 ..........<..... - 1a00 00004e08 00000000 00000100 55000000 ..N.........U... - 1a10 00000000 00000000 00000000 000a014e ...............N - 1a20 08000000 00000060 08000000 00000001 .......`........ - 1a30 00520000 00000000 00000000 00000000 .R.............. - 1a40 00000a00 00000001 60080000 00000000 ........`....... - 1a50 63080000 00000000 01005a63 08000000 c.........Zc.... - 1a60 0000006d 08000000 00000001 00516d08 ...m.........Qm. - 1a70 00000000 00006d08 00000000 00000100 ......m......... - 1a80 5a000000 00000000 00000000 00000000 Z............... - 1a90 00000000 00d90800 00000000 00e40800 ................ - 1aa0 00000000 00010051 e4080000 00000000 .......Q........ - 1ab0 e5080000 00000000 01005f00 00000000 .........._..... - 1ac0 00000000 00000000 00000000 00d90800 ................ - 1ad0 00000000 00e50800 00000000 0001005e ...............^ - 1ae0 00000000 00000000 00000000 00000000 ................ - 1af0 0000d908 00000000 0000e508 00000000 ................ - 1b00 00000400 91f87e9f 00000000 00000000 ......~......... - 1b10 00000000 00000000 02000000 0000f50b ................ - 1b20 00000000 00000e0c 00000000 00000b00 ................ - 1b30 407d001c 0cffffff ff1a9f0e 0c000000 @}.............. - 1b40 00000015 0c000000 00000001 0051150c .............Q.. - 1b50 00000000 0000160c 00000000 00000b00 ................ - 1b60 407d001c 0cffffff ff1a9f00 00000000 @}.............. - 1b70 00000000 00000000 00000002 00f50b00 ................ - 1b80 00000000 00160c00 00000000 00030073 ...............s - 1b90 089f0000 00000000 00000000 00000000 ................ - 1ba0 00000200 0000f50b 00000000 0000150c ................ - 1bb0 00000000 00000900 91007000 2208881c ..........p."... - 1bc0 9f150c00 00000000 00160c00 00000000 ................ - 1bd0 000f007d 000cffff ffff1a91 00220888 ...}.........".. - 1be0 1c9f0000 00000000 00000000 00000000 ................ - 1bf0 00000201 01010101 0100160c 00000000 ................ - 1c00 0000480c 00000000 00000400 91f87e9f ..H...........~. - 1c10 480c0000 00000000 610c0000 00000000 H.......a....... - 1c20 040091fc 7e9f610c 00000000 00007a0c ....~.a.......z. - 1c30 00000000 00000400 91807f9f 7a0c0000 ............z... - 1c40 00000000 980c0000 00000000 04009184 ................ - 1c50 7f9f0000 00000000 00000000 00000000 ................ - 1c60 00000902 160c0000 00000000 390c0000 ............9... - 1c70 00000000 030091f8 7e000000 00000000 ........~....... - 1c80 00000000 00000000 00090000 01010216 ................ - 1c90 0c000000 00000033 0c000000 00000003 .......3........ - 1ca0 0091e87e 330c0000 00000000 390c0000 ...~3.......9... - 1cb0 00000000 01005039 0c000000 00000039 ......P9.......9 - 1cc0 0c000000 0000000a 00700011 b1f3ddf1 .........p...... - 1cd0 791e9f00 00000000 00000000 00000000 y............... - 1ce0 00000008 02480c00 00000000 00570c00 .....H.......W.. - 1cf0 00000000 00030091 fc7e0000 00000000 .........~...... - 1d00 00000000 00000000 00000800 00010102 ................ - 1d10 480c0000 00000000 540c0000 00000000 H.......T....... - 1d20 030091ec 7e540c00 00000000 00570c00 ....~T.......W.. - 1d30 00000000 00010050 570c0000 00000000 .......PW....... - 1d40 570c0000 00000000 0a007000 11b1f3dd W.........p..... - 1d50 f1791e9f 00000000 00000000 00000000 .y.............. - 1d60 00000000 0802610c 00000000 0000700c ......a.......p. - 1d70 00000000 00000300 91807f00 00000000 ................ - 1d80 00000000 00000000 00000008 00000101 ................ - 1d90 02610c00 00000000 006d0c00 00000000 .a.......m...... - 1da0 00030091 f07e6d0c 00000000 0000700c .....~m.......p. - 1db0 00000000 00000100 50700c00 00000000 ........Pp...... - 1dc0 00700c00 00000000 000a0070 0011b1f3 .p.........p.... - 1dd0 ddf1791e 9f000000 00000000 00000000 ..y............. - 1de0 00000000 0008027a 0c000000 00000089 .......z........ - 1df0 0c000000 00000003 0091847f 00000000 ................ - 1e00 00000000 00000000 00000000 08000001 ................ - 1e10 01027a0c 00000000 0000860c 00000000 ..z............. - 1e20 00000300 91f47e86 0c000000 00000089 ......~......... - 1e30 0c000000 00000001 0050890c 00000000 .........P...... - 1e40 0000890c 00000000 00000a00 700011b1 ............p... - 1e50 f3ddf179 1e9f0000 00000000 00000000 ...y............ - 1e60 00000000 00000200 690d0000 00000000 ........i....... - 1e70 7e0d0000 00000000 0200389f 00000000 ~.........8..... - 1e80 00000000 00000000 00000000 0200690d ..............i. - 1e90 00000000 00007e0d 00000000 00000300 ......~......... - 1ea0 73089f00 00000000 00000000 00000000 s............... - 1eb0 00000002 00690d00 00000000 007e0d00 .....i.......~.. - 1ec0 00000000 00090091 00700022 08881c9f .........p.".... - 1ed0 00000000 00000000 00000000 00000000 ................ - 1ee0 00020000 0000ef08 00000000 0000f908 ................ - 1ef0 00000000 00000200 389f2c09 00000000 ........8.,..... - 1f00 00003a0a 00000000 00000200 389f2b0b ..:.........8.+. - 1f10 00000000 0000f50b 00000000 00000200 ................ - 1f20 389f0000 00000000 00000000 00000000 8............... - 1f30 00000002 00000000 00000000 0001ef08 ................ - 1f40 00000000 0000f908 00000000 00000100 ................ - 1f50 5e2c0900 00000000 00590900 00000000 ^,.......Y...... - 1f60 0001005e fe090000 00000000 110a0000 ...^............ - 1f70 00000000 08007e00 70002223 109f110a ......~.p."#.... - 1f80 00000000 00001e0a 00000000 00000100 ................ - 1f90 5e5d0b00 00000000 00a10b00 00000000 ^].............. - 1fa0 0001005e a10b0000 00000000 f00b0000 ...^............ - 1fb0 00000000 03007308 9f000000 00000000 ......s......... - 1fc0 00000000 00000000 00000200 000000ef ................ - 1fd0 08000000 000000f9 08000000 00000003 ................ - 1fe0 0091c87e 2c090000 00000000 3a0a0000 ...~,.......:... - 1ff0 00000000 030091c8 7e2b0b00 00000000 ........~+...... - 2000 00f50b00 00000000 00030091 c87e0000 .............~.. - 2010 00000000 00000000 00000000 00000002 ................ - 2020 04000000 00030300 00000001 ef080000 ................ - 2030 00000000 f9080000 00000000 01005e2c ..............^, - 2040 09000000 00000059 09000000 00000001 .......Y........ - 2050 005e8009 00000000 00008a09 00000000 .^.............. - 2060 00000100 508a0900 00000000 00a90900 ....P........... - 2070 00000000 00030070 709fcf09 00000000 .......pp....... - 2080 0000e209 00000000 00000100 505d0b00 ............P].. - 2090 00000000 00a10b00 00000000 0001005e ...............^ - 20a0 a10b0000 00000000 f00b0000 00000000 ................ - 20b0 03007308 9f000000 00000000 00000000 ..s............. - 20c0 00000000 00000200 00000000 000000ef ................ - 20d0 08000000 000000f9 08000000 00000001 ................ - 20e0 005f3009 00000000 00003a0a 00000000 ._0.......:..... - 20f0 00000100 5f2b0b00 00000000 00380b00 ...._+.......8.. - 2100 00000000 0001005f 380b0000 00000000 ......._8....... - 2110 5d0b0000 00000000 03007310 9f5d0b00 ].........s..].. - 2120 00000000 00f50b00 00000000 0001005f ..............._ - 2130 00000000 00000000 00000000 00000000 ................ - 2140 0200ef08 00000000 0000f908 00000000 ................ - 2150 00000200 389f0000 00000000 00000000 ....8........... - 2160 00000000 00000200 ef080000 00000000 ................ - 2170 f9080000 00000000 01005e00 00000000 ..........^..... - 2180 00000000 00000000 00000002 000000ef ................ - 2190 08000000 000000f4 08000000 00000009 ................ - 21a0 00910070 00220888 1c9ff408 00000000 ...p.".......... - 21b0 0000f908 00000000 00000900 91007100 ..............q. - 21c0 2208881c 9f000000 00000000 00000000 "............... - 21d0 00000000 00080000 01800900 00000000 ................ - 21e0 008a0900 00000000 00020070 008a0900 ...........p.... - 21f0 00000000 00a90900 00000000 00020070 ...............p - 2200 70000000 00000000 00000000 00000000 p............... - 2210 00080180 09000000 000000a9 09000000 ................ - 2220 00000001 00510000 00000000 00000000 .....Q.......... - 2230 00000000 00000d0e 0e0fc209 00000000 ................ - 2240 0000c209 00000000 00000100 54c20900 ............T... - 2250 00000000 00c20900 00000000 00010058 ...............X - 2260 00000000 00000000 00000000 00000000 ................ - 2270 0a01a909 00000000 0000c209 00000000 ................ - 2280 00000100 55000000 00000000 00000000 ....U........... - 2290 00000000 00180000 000001c2 09000000 ................ - 22a0 000000c5 09000000 00000001 0059c509 .............Y.. - 22b0 00000000 0000cf09 00000000 00000100 ................ - 22c0 52cf0900 00000000 00cf0900 00000000 R............... - 22d0 00010059 00000000 00000000 00000000 ...Y............ - 22e0 00000000 0000380b 00000000 00004a0b ......8.......J. - 22f0 00000000 00000100 5f000000 00000000 ........_....... - 2300 00000000 00000000 00000038 0b000000 ...........8.... - 2310 0000004a 0b000000 00000004 0091f87e ...J...........~ - 2320 9f000000 00000000 00000000 00000000 ................ - 2330 00020000 005d0b00 00000000 00710b00 .....].......q.. - 2340 00000000 000b0040 71001c0c ffffffff .......@q....... - 2350 1a9f710b 00000000 00007a0b 00000000 ..q.......z..... - 2360 00000b00 4070001c 0cffffff ff1a9f00 ....@p.......... - 2370 00000000 00000000 00000000 00000002 ................ - 2380 005d0b00 00000000 007b0b00 00000000 .].......{...... - 2390 0001005e 00000000 00000000 00000000 ...^............ - 23a0 00000000 02005d0b 00000000 00007a0b ......].......z. - 23b0 00000000 00000900 91007000 2208881c ..........p."... - 23c0 9f000000 00000000 00000000 00000000 ................ - 23d0 00020101 01010101 007b0b00 00000000 .........{...... - 23e0 00a50b00 00000000 00040091 f87e9fa5 .............~.. - 23f0 0b000000 000000be 0b000000 00000004 ................ - 2400 0091fc7e 9fbe0b00 00000000 00d70b00 ...~............ - 2410 00000000 00040091 807f9fd7 0b000000 ................ - 2420 000000f5 0b000000 00000004 0091847f ................ - 2430 9f000000 00000000 00000000 00000000 ................ - 2440 0009027b 0b000000 0000008c 0b000000 ...{............ - 2450 00000003 0091f87e 00000000 00000000 .......~........ - 2460 00000000 00000000 09000001 01027b0b ..............{. - 2470 00000000 0000890b 00000000 00000300 ................ - 2480 91e87e89 0b000000 0000008c 0b000000 ..~............. - 2490 00000001 00508c0b 00000000 00008c0b .....P.......... - 24a0 00000000 00000a00 700011b1 f3ddf179 ........p......y - 24b0 1e9f0000 00000000 00000000 00000000 ................ - 24c0 00000802 a50b0000 00000000 b40b0000 ................ - 24d0 00000000 030091fc 7e000000 00000000 ........~....... - 24e0 00000000 00000000 00080000 010102a5 ................ - 24f0 0b000000 000000b1 0b000000 00000003 ................ - 2500 0091ec7e b10b0000 00000000 b40b0000 ...~............ - 2510 00000000 010050b4 0b000000 000000b4 ......P......... - 2520 0b000000 0000000a 00700011 b1f3ddf1 .........p...... - 2530 791e9f00 00000000 00000000 00000000 y............... - 2540 00000008 02be0b00 00000000 00cd0b00 ................ - 2550 00000000 00030091 807f0000 00000000 ................ - 2560 00000000 00000000 00000800 00010102 ................ - 2570 be0b0000 00000000 ca0b0000 00000000 ................ - 2580 030091f0 7eca0b00 00000000 00cd0b00 ....~........... - 2590 00000000 00010050 cd0b0000 00000000 .......P........ - 25a0 cd0b0000 00000000 0a007000 11b1f3dd ..........p..... - 25b0 f1791e9f 00000000 00000000 00000000 .y.............. - 25c0 00000000 0802d70b 00000000 0000e60b ................ - 25d0 00000000 00000300 91847f00 00000000 ................ - 25e0 00000000 00000000 00000008 00000101 ................ - 25f0 02d70b00 00000000 00e30b00 00000000 ................ - 2600 00030091 f47ee30b 00000000 0000e60b .....~.......... - 2610 00000000 00000100 50e60b00 00000000 ........P....... - 2620 00e60b00 00000000 000a0070 0011b1f3 ...........p.... - 2630 ddf1791e 9f000000 00000000 00000000 ..y............. - 2640 00000000 00000000 00000000 006b0300 .............k.. - 2650 00000000 00890300 00000000 00010051 ...............Q - 2660 89030000 00000000 ed030000 00000000 ................ - 2670 030091a0 7fed0300 00000000 00ba0d00 ................ - 2680 00000000 000400f3 01519fba 0d000000 .........Q...... - 2690 000000ea 0d000000 00000003 0091a07f ................ - 26a0 00000000 00000000 00000000 00000000 ................ - 26b0 00000000 00000000 00000000 6b030000 ............k... - 26c0 00000000 e8030000 00000000 040091e0 ................ - 26d0 7e9fe803 00000000 0000ed03 00000000 ~............... - 26e0 00000100 50ed0300 00000000 00ba0d00 ....P........... - 26f0 00000000 00030091 c87eba0d 00000000 .........~...... - 2700 0000cd0d 00000000 00000400 91e07e9f ..............~. - 2710 cd0d0000 00000000 dd0d0000 00000000 ................ - 2720 010050dd 0d000000 000000ea 0d000000 ..P............. - 2730 00000003 0091c87e 00000000 00000000 .......~........ - 2740 00000000 00000000 0100a003 00000000 ................ - 2750 0000c003 00000000 00000300 082c9f00 .............,.. - 2760 00000000 00000000 00000000 00000001 ................ - 2770 00a00300 00000000 00c00300 00000000 ................ - 2780 00040091 907f9f00 00000000 00000000 ................ - 2790 00000000 00000001 00a00300 00000000 ................ - 27a0 00c00300 00000000 00040091 e07e9f00 .............~.. - 27b0 00000000 00000000 00000000 00000001 ................ - 27c0 01000001 00000067 06000000 0000008e .......g........ - 27d0 07000000 00000003 0091c87e 040b0000 ...........~.... - 27e0 00000000 2b0b0000 00000000 030091c8 ....+........... - 27f0 7ed20d00 00000000 00dd0d00 00000000 ~............... - 2800 00010050 dd0d0000 00000000 ea0d0000 ...P............ - 2810 00000000 030091c8 7e000000 00000000 ........~....... - 2820 00000000 00000000 00000000 00000079 ...............y - 2830 06000000 0000007d 06000000 00000001 .......}........ - 2840 00507d06 00000000 0000a106 00000000 .P}............. - 2850 00000100 59260b00 00000000 002b0b00 ....Y&.......+.. - 2860 00000000 00010050 00000000 00000000 .......P........ - 2870 00000000 00000000 02017d06 00000000 ..........}..... - 2880 00008e07 00000000 00000200 309f0000 ............0... - 2890 00000000 00000000 00000000 00000100 ................ - 28a0 00020202 00020200 00020200 00020203 ................ - 28b0 7d060000 00000000 88060000 00000000 }............... - 28c0 0c009188 7f94040c ffffffff 1a9f8806 ................ - 28d0 00000000 0000ac06 00000000 00000100 ................ - 28e0 50ac0600 00000000 00cc0600 00000000 P............... - 28f0 00010052 00070000 00000000 1c070000 ...R............ - 2900 00000000 0100501c 07000000 00000020 ......P........ - 2910 07000000 00000003 00707f9f 20070000 .........p.. ... - 2920 00000000 3a070000 00000000 0100503a ....:.........P: - 2930 07000000 00000040 07000000 00000005 .......@........ - 2940 00700031 2e9f4007 00000000 00005807 [email protected]. - 2950 00000000 00000200 319f5807 00000000 ........1.X..... - 2960 00005807 00000000 00000200 309f0000 ..X.........0... - 2970 00000000 00000000 00000000 00000107 ................ - 2980 07070707 07030002 02020202 02037d06 ..............}. - 2990 00000000 00009206 00000000 00000400 ................ - 29a0 91f87e9f 92060000 00000000 b2060000 ..~............. - 29b0 00000000 040091fc 7e9fb206 00000000 ........~....... - 29c0 0000d206 00000000 00000400 91807f9f ................ - 29d0 d2060000 00000000 e8060000 00000000 ................ - 29e0 04009184 7f9f0007 00000000 00000507 ................ - 29f0 00000000 00000100 54050700 00000000 ........T....... - 2a00 00220700 00000000 00030074 019f2207 .".........t..". - 2a10 00000000 00004007 00000000 00000300 ......@......... - 2a20 74029f40 07000000 00000058 07000000 [email protected].... - 2a30 00000003 0074039f 00000000 00000000 .....t.......... - 2a40 00000000 00000000 01050500 00000005 ................ - 2a50 05000000 00050500 00000002 02000000 ................ - 2a60 00020200 00020200 00007d06 00000000 ..........}..... - 2a70 00009206 00000000 00000100 59920600 ............Y... - 2a80 00000000 00a10600 00000000 00100091 ................ - 2a90 f87e9404 11bddcca 957c1e79 00229fa1 .~.......|.y.".. - 2aa0 06000000 000000a5 06000000 00000001 ................ - 2ab0 0059ac06 00000000 0000b206 00000000 .Y.............. - 2ac0 00000100 59b20600 00000000 00c10600 ....Y........... - 2ad0 00000000 00100091 fc7e9404 11bddcca .........~...... - 2ae0 957c1e79 00229fc1 06000000 000000c5 .|.y.".......... - 2af0 06000000 00000001 0059cc06 00000000 .........Y...... - 2b00 0000d206 00000000 00000100 59d20600 ............Y... - 2b10 00000000 00dd0600 00000000 00100091 ................ - 2b20 807f9404 11bddcca 957c1e79 00229fdd .........|.y.".. - 2b30 06000000 000000e1 06000000 00000001 ................ - 2b40 0059e806 00000000 00000507 00000000 .Y.............. - 2b50 00000100 59050700 00000000 00110700 ....Y........... - 2b60 00000000 00110074 00940108 ff1a0cb1 .......t........ - 2b70 6756161e 7900229f 11070000 00000000 gV..y."......... - 2b80 15070000 00000000 0100591c 07000000 ..........Y..... - 2b90 00000022 07000000 00000001 00592207 ...".........Y". - 2ba0 00000000 00002f07 00000000 00001100 ....../......... - 2bb0 74019401 08ff1a0c b1675616 1e790022 t........gV..y." - 2bc0 9f3a0700 00000000 00400700 00000000 .:.......@...... - 2bd0 00010059 40070000 00000000 4d070000 [email protected]... - 2be0 00000000 11007402 940108ff 1a0cb167 ......t........g - 2bf0 56161e79 00229f58 07000000 0000006a V..y.".X.......j - 2c00 07000000 00000001 00590000 00000000 .........Y...... - 2c10 00000000 00000000 00000205 02050205 ................ - 2c20 92060000 00000000 92060000 00000000 ................ - 2c30 0200309f b2060000 00000000 b2060000 ..0............. - 2c40 00000000 0200309f d2060000 00000000 ......0......... - 2c50 d2060000 00000000 0200309f 00000000 ..........0..... - 2c60 00000000 00000000 00000000 02050205 ................ - 2c70 02059206 00000000 00009206 00000000 ................ - 2c80 00000400 91f87e9f b2060000 00000000 ......~......... - 2c90 b2060000 00000000 040091fc 7e9fd206 ............~... - 2ca0 00000000 0000d206 00000000 00000400 ................ - 2cb0 91807f9f 00000000 00000000 00000000 ................ - 2cc0 00000000 04000000 00015807 00000000 ..........X..... - 2cd0 00007507 00000000 00000100 59750700 ..u.........Yu.. - 2ce0 00000000 008e0700 00000000 00010050 ...............P - 2cf0 8e070000 00000000 8e070000 00000000 ................ - 2d00 06007000 7100279f 00000000 00000000 ..p.q.'......... - 2d10 00000000 00000000 00000000 00000000 ................ - 2d20 a0020000 00000000 ba020000 00000000 ................ - 2d30 010055ba 02000000 000000c8 02000000 ..U............. - 2d40 00000001 0053c802 00000000 0000d302 .....S.......... - 2d50 00000000 00000300 73709fd3 02000000 ........sp...... - 2d60 0000004d 03000000 00000004 00f30155 ...M...........U - 2d70 9f000000 00000000 00000000 00000000 ................ - 2d80 00000000 000000a0 02000000 000000aa ................ - 2d90 02000000 00000001 0054aa02 00000000 .........T...... - 2da0 0000d302 00000000 00000100 56d30200 ............V... - 2db0 00000000 004d0300 00000000 000400f3 .....M.......... - 2dc0 01549f00 00000000 00000000 00000000 .T.............. - 2dd0 00000000 000000a0 02000000 000000be ................ - 2de0 02000000 00000001 0051be02 00000000 .........Q...... - 2df0 00004d03 00000000 00000400 f301519f ..M...........Q. - 2e00 00000000 00000000 00000000 00000000 ................ - 2e10 0200bf02 00000000 0000d302 00000000 ................ - 2e20 00000200 309f0000 00000000 00000000 ....0........... - 2e30 00000000 00000000 00006002 00000000 ..........`..... - 2e40 00006a02 00000000 00000100 546a0200 ..j.........Tj.. - 2e50 00000000 00870200 00000000 000400f3 ................ - 2e60 01549f00 00000000 00000000 00000000 .T.............. - 2e70 00000000 00000060 02000000 0000006a .......`.......j - 2e80 02000000 00000001 00516a02 00000000 .........Qj..... - 2e90 00008702 00000000 00000400 f301519f ..............Q. - 2ea0 00000000 00000000 00000000 00000000 ................ - 2eb0 00006202 00000000 00008702 00000000 ..b............. - 2ec0 00000100 50000000 00000000 00000000 ....P........... - 2ed0 00000000 00020000 00000162 02000000 ...........b.... - 2ee0 0000006a 02000000 00000002 00309f6a ...j.........0.j - 2ef0 02000000 00000077 02000000 00000007 .......w........ - 2f00 007400f3 01541c9f 77020000 00000000 .t...T..w....... - 2f10 7f020000 00000000 0800f301 54207400 ............T t. - 2f20 229f0000 00000000 00000000 00000000 "............... - 2f30 00000000 00000000 00000000 00000000 ................ - 2f40 0000c001 00000000 0000d201 00000000 ................ - 2f50 00000100 55d20100 00000000 00f80100 ....U........... - 2f60 00000000 00010053 f8010000 00000000 .......S........ - 2f70 fa010000 00000000 0400f301 559ffa01 ............U... - 2f80 00000000 00000902 00000000 00000100 ................ - 2f90 53090200 00000000 000b0200 00000000 S............... - 2fa0 000400f3 01559f0b 02000000 0000001d .....U.......... - 2fb0 02000000 00000001 00531d02 00000000 .........S...... - 2fc0 00002302 00000000 00000400 f301559f ..#...........U. - 2fd0 23020000 00000000 48020000 00000000 #.......H....... - 2fe0 01005300 00000000 00000000 00000000 ..S............. - 2ff0 00000000 00000000 00000000 00000000 ................ - 3000 000000c0 01000000 000000cf 01000000 ................ - 3010 00000001 0054cf01 00000000 0000f901 .....T.......... - 3020 00000000 00000100 56f90100 00000000 ........V....... - 3030 00fa0100 00000000 000400f3 01549ffa .............T.. - 3040 01000000 0000000a 02000000 00000001 ................ - 3050 00560a02 00000000 00000b02 00000000 .V.............. - 3060 00000400 f301549f 0b020000 00000000 ......T......... - 3070 1e020000 00000000 0100561e 02000000 ..........V..... - 3080 00000023 02000000 00000004 00f30154 ...#...........T - 3090 9f230200 00000000 00480200 00000000 .#.......H...... - 30a0 00010056 00000000 00000000 00000000 ...V............ - 30b0 00000000 02000000 00000000 00000000 ................ - 30c0 00000000 c0010000 00000000 d2010000 ................ - 30d0 00000000 010055d2 01000000 000000f8 ......U......... - 30e0 01000000 00000001 0053f801 00000000 .........S...... - 30f0 0000fa01 00000000 00000400 f301559f ..............U. - 3100 fa010000 00000000 09020000 00000000 ................ - 3110 01005309 02000000 0000000b 02000000 ..S............. - 3120 00000004 00f30155 9f0b0200 00000000 .......U........ - 3130 001d0200 00000000 00010053 1d020000 ...........S.... - 3140 00000000 23020000 00000000 0400f301 ....#........... - 3150 559f2302 00000000 00004802 00000000 U.#.......H..... - 3160 00000100 53000000 00000000 00000000 ....S........... - 3170 00000000 00030000 00000000 00000000 ................ - 3180 00000000 00c00100 00000000 00cf0100 ................ - 3190 00000000 00010054 cf010000 00000000 .......T........ - 31a0 f9010000 00000000 010056f9 01000000 ..........V..... - 31b0 000000fa 01000000 00000004 00f30154 ...............T - 31c0 9ffa0100 00000000 000a0200 00000000 ................ - 31d0 00010056 0a020000 00000000 0b020000 ...V............ - 31e0 00000000 0400f301 549f0b02 00000000 ........T....... - 31f0 00001e02 00000000 00000100 561e0200 ............V... - 3200 00000000 00230200 00000000 000400f3 .....#.......... - 3210 01549f23 02000000 00000048 02000000 .T.#.......H.... - 3220 00000001 00560000 00000000 00000000 .....V.......... - 3230 00000000 00000000 00000000 00000000 ................ - 3240 d7010000 00000000 f3010000 00000000 ................ - 3250 010050fa 01000000 00000006 02000000 ..P............. - 3260 00000001 00500b02 00000000 00002202 .....P........". - 3270 00000000 00000100 50230200 00000000 ........P#...... - 3280 002c0200 00000000 00010050 31020000 .,.........P1... - 3290 00000000 46020000 00000000 01005000 ....F.........P. - 32a0 00000000 00000000 00000000 00000000 ................ - 32b0 00000000 00000010 00000000 0000002f .............../ - 32c0 00000000 00000001 00552f00 00000000 .........U/..... - 32d0 00003b00 00000000 00000400 f301559f ..;...........U. - 32e0 3b000000 00000000 50000000 00000000 ;.......P....... - 32f0 01005550 00000000 000000d8 00000000 ..UP............ - 3300 00000004 00f30155 9f000000 00000000 .......U........ - 3310 00000000 00000000 00000000 00000000 ................ - 3320 00100000 00000000 002f0000 00000000 ........./...... - 3330 00010054 2f000000 00000000 3b000000 ...T/.......;... - 3340 00000000 0400f301 549f3b00 00000000 ........T.;..... - 3350 00005000 00000000 00000100 54500000 ..P.........TP.. - 3360 00000000 00d80000 00000000 000400f3 ................ - 3370 01549f00 00000000 00000000 00000000 .T.............. - 3380 00000000 00000050 00000000 000000af .......P........ - 3390 00000000 00000001 005cc100 00000000 .........\...... - 33a0 0000d800 00000000 00000100 5c000000 ............\... - 33b0 00000000 00000000 00000000 00000000 ................ - 33c0 00020000 00500000 00000000 00780000 .....P.......x.. - 33d0 00000000 00010056 78000000 00000000 .......Vx....... - 33e0 81000000 00000000 03007668 9f810000 ..........vh.... - 33f0 00000000 00af0000 00000000 00010056 ...............V - 3400 c1000000 00000000 d8000000 00000000 ................ - 3410 01005600 00000000 00000000 00000000 ..V............. - 3420 00000000 00000000 00000000 00000000 ................ - 3430 00e00000 00000000 00ef0000 00000000 ................ - 3440 00010055 ef000000 00000000 22010000 ...U........"... - 3450 00000000 01005622 01000000 00000023 ......V".......# - 3460 01000000 00000004 00f30155 9f230100 ...........U.#.. - 3470 00000000 00380100 00000000 00010056 .....8.........V - 3480 38010000 00000000 39010000 00000000 8.......9....... - 3490 0400f301 559f3901 00000000 00004e01 ....U.9.......N. - 34a0 00000000 00000100 564e0100 00000000 ........VN...... - 34b0 004f0100 00000000 000400f3 01559f00 .O...........U.. - 34c0 00000000 00000000 00000000 00000000 ................ - 34d0 00000000 00000000 00000000 00e00000 ................ - 34e0 00000000 00f30000 00000000 00010054 ...............T - 34f0 f3000000 00000000 21010000 00000000 ........!....... - 3500 01005321 01000000 00000023 01000000 ..S!.......#.... - 3510 00000004 00f30154 9f230100 00000000 .......T.#...... - 3520 00370100 00000000 00010053 37010000 .7.........S7... - 3530 00000000 39010000 00000000 0400f301 ....9........... - 3540 549f3901 00000000 00004d01 00000000 T.9.......M..... - 3550 00000100 534d0100 00000000 004f0100 ....SM.......O.. - 3560 00000000 000400f3 01549f00 00000000 .........T...... - 3570 00000000 00000000 00000000 010f0100 ................ - 3580 00000000 001c0100 00000000 00010053 ...............S - 3590 00000000 00000000 00000000 00000000 ................ - 35a0 00010f01 00000000 00001c01 00000000 ................ - 35b0 00000100 56000000 00000000 00000000 ....V........... - 35c0 00000000 00000000 00000000 00000050 ...............P - 35d0 01000000 00000071 01000000 00000001 .......q........ - 35e0 00557101 00000000 0000a801 00000000 .Uq............. - 35f0 00000100 5ca80100 00000000 00ac0100 ....\........... - 3600 00000000 00010055 ac010000 00000000 .......U........ - 3610 ad010000 00000000 0400f301 559fad01 ............U... - 3620 00000000 0000b101 00000000 00000100 ................ - 3630 55000000 00000000 00000000 00000000 U............... - 3640 00000000 00000050 01000000 00000071 .......P.......q - 3650 01000000 00000001 00547101 00000000 .........Tq..... - 3660 0000ad01 00000000 00000400 f301549f ..............T. - 3670 ad010000 00000000 b1010000 00000000 ................ - 3680 01005400 00000000 00000000 00000000 ..T............. - 3690 00000000 005a0100 00000000 00710100 .....Z.......q.. - 36a0 00000000 00020030 9f000000 00000000 .......0........ - 36b0 00000000 00000000 00000000 00010000 ................ - 36c0 00710100 00000000 007c0100 00000000 .q.......|...... - 36d0 00010053 7c010000 00000000 81010000 ...S|........... - 36e0 00000000 03007368 9f810100 00000000 ......sh........ - 36f0 00970100 00000000 00010053 97010000 ...........S.... - 3700 00000000 a1010000 00000000 03007368 ..............sh - 3710 9f000000 00000000 00000000 00000000 ................ - 3720 00000000 00000000 00000000 00000000 ................ - 3730 10000000 00000029 10000000 00000001 .......)........ - 3740 00552910 00000000 00007b10 00000000 .U).......{..... - 3750 00000100 567b1000 00000000 00c91200 ....V{.......... - 3760 00000000 000400f3 01559fc9 12000000 .........U...... - 3770 000000d0 12000000 00000001 0055d012 .............U.. - 3780 00000000 00003113 00000000 00000400 ......1......... - 3790 f301559f 31130000 00000000 38130000 ..U.1.......8... - 37a0 00000000 01005638 13000000 0000003d ......V8.......= - 37b0 13000000 00000004 00f30155 9f000000 ...........U.... - 37c0 00000000 00000000 00000000 00000000 ................ - 37d0 00000000 00000000 00001000 00000000 ................ - 37e0 00301000 00000000 00010054 30100000 .0.........T0... - 37f0 00000000 5c100000 00000000 0100535c ....\.........S\ - 3800 10000000 000000c9 12000000 00000004 ................ - 3810 00f30154 9fc91200 00000000 00d41200 ...T............ - 3820 00000000 00010054 d4120000 00000000 .......T........ - 3830 31130000 00000000 0400f301 549f3113 1...........T.1. - 3840 00000000 00003d13 00000000 00000100 ......=......... - 3850 53000000 00000000 00000000 00000000 S............... - 3860 00000000 00000000 00000000 10000000 ................ - 3870 00000030 10000000 00000001 00513010 ...0.........Q0. - 3880 00000000 00007b10 00000000 00000300 ......{......... - 3890 91b87f7b 10000000 000000c9 12000000 ...{............ - 38a0 00000004 00f30151 9fc91200 00000000 .......Q........ - 38b0 00d41200 00000000 00010051 d4120000 ...........Q.... - 38c0 00000000 3d130000 00000000 030091b8 ....=........... - 38d0 7f000000 00000000 00000000 00000000 ................ - 38e0 00000000 00000000 00000000 10000000 ................ - 38f0 00000030 10000000 00000001 00523010 ...0.........R0. - 3900 00000000 00007b10 00000000 00000300 ......{......... - 3910 91b07f7b 10000000 000000c9 12000000 ...{............ - 3920 00000004 00f30152 9fc91200 00000000 .......R........ - 3930 00d41200 00000000 00010052 d4120000 ...........R.... - 3940 00000000 3d130000 00000000 030091b0 ....=........... - 3950 7f000000 00000000 00000000 00000000 ................ - 3960 00000000 00000000 002c1000 00000000 .........,...... - 3970 00301000 00000000 00010052 30100000 .0.........R0... - 3980 00000000 7b100000 00000000 030091b0 ....{........... - 3990 7f7b1000 00000000 00c91200 00000000 .{.............. - 39a0 000400f3 01529f31 13000000 0000003d .....R.1.......= - 39b0 13000000 00000003 0091b07f 00000000 ................ - 39c0 00000000 00000000 00000000 00000000 ................ - 39d0 00000000 2c100000 00000000 30100000 ....,.......0... - 39e0 00000000 01005130 10000000 0000007b ......Q0.......{ - 39f0 10000000 00000003 0091b87f 7b100000 ............{... - 3a00 00000000 c9120000 00000000 0400f301 ................ - 3a10 519f3113 00000000 00003d13 00000000 Q.1.......=..... - 3a20 00000300 91b87f00 00000000 00000000 ................ - 3a30 00000000 00000000 00000000 0000002c ..............., - 3a40 10000000 00000030 10000000 00000001 .......0........ - 3a50 00543010 00000000 00005c10 00000000 .T0.......\..... - 3a60 00000100 535c1000 00000000 00c91200 ....S\.......... - 3a70 00000000 000400f3 01549f31 13000000 .........T.1.... - 3a80 0000003d 13000000 00000001 00530000 ...=.........S.. - 3a90 00000000 00000000 00000000 00000000 ................ - 3aa0 00000000 00002c10 00000000 00007b10 ......,.......{. - 3ab0 00000000 00000100 567b1000 00000000 ........V{...... - 3ac0 00c91200 00000000 000400f3 01559f31 .............U.1 - 3ad0 13000000 00000038 13000000 00000001 .......8........ - 3ae0 00563813 00000000 00003d13 00000000 .V8.......=..... - 3af0 00000400 f301559f 00000000 00000000 ......U......... - 3b00 00000000 00000000 03000000 2c100000 ............,... - 3b10 00000000 7b100000 00000000 0200309f ....{.........0. - 3b20 31130000 00000000 3d130000 00000000 1.......=....... - 3b30 0200309f 00000000 00000000 00000000 ..0............. - 3b40 00000000 04000000 00000000 00000000 ................ - 3b50 00000000 00000000 00002c10 00000000 ..........,..... - 3b60 00007b10 00000000 00000200 309f7b10 ..{.........0.{. - 3b70 00000000 00003011 00000000 00000300 ......0......... - 3b80 91947f78 11000000 000000b9 11000000 ...x............ - 3b90 00000003 0091947f b9110000 00000000 ................ - 3ba0 c1110000 00000000 010050c1 11000000 ..........P..... - 3bb0 00000032 12000000 00000003 0091947f ...2............ - 3bc0 32120000 00000000 3a120000 00000000 2.......:....... - 3bd0 0100503a 12000000 00000078 12000000 ..P:.......x.... - 3be0 00000003 0091947f 7d120000 00000000 ........}....... - 3bf0 a2120000 00000000 03009194 7fa21200 ................ - 3c00 00000000 00aa1200 00000000 00010050 ...............P - 3c10 aa120000 00000000 c9120000 00000000 ................ - 3c20 03009194 7f311300 00000000 003d1300 .....1.......=.. - 3c30 00000000 00020030 9f000000 00000000 .......0........ - 3c40 00000000 00000000 00050001 00000000 ................ - 3c50 0000002c 10000000 0000007b 10000000 ...,.......{.... - 3c60 00000002 00309f7b 10000000 000000a7 .....0.{........ - 3c70 10000000 00000003 0091f87e 78110000 ...........~x... - 3c80 00000000 78120000 00000000 030091f8 ....x........... - 3c90 7e7d1200 00000000 00c91200 00000000 ~}.............. - 3ca0 00030091 f87e3113 00000000 00003d13 .....~1.......=. - 3cb0 00000000 00000200 309f0000 00000000 ........0....... - 3cc0 00000000 00000000 00000600 00002c10 ..............,. - 3cd0 00000000 0000c912 00000000 00000400 ................ - 3ce0 0a00049f 31130000 00000000 3d130000 ....1.......=... - 3cf0 00000000 04000a00 049f0000 00000000 ................ - 3d00 00000000 00000000 00000700 00000000 ................ - 3d10 00000000 00002c10 00000000 00007b10 ......,.......{. - 3d20 00000000 00000400 0a00049f 7b100000 ............{... - 3d30 00000000 c7100000 00000000 010053c7 ..............S. - 3d40 10000000 000000f1 10000000 00000001 ................ - 3d50 005cf110 00000000 00003011 00000000 .\........0..... - 3d60 00000100 53781100 00000000 00c91200 ....Sx.......... - 3d70 00000000 00010053 31130000 00000000 .......S1....... - 3d80 3d130000 00000000 04000a00 049f0000 =............... - 3d90 00000000 00000000 00000000 00000000 ................ - 3da0 00000000 00000000 00000000 00000000 ................ - 3db0 00000000 00000000 39100000 00000000 ........9....... - 3dc0 3d100000 00000000 0100503d 10000000 =.........P=.... - 3dd0 0000006f 11000000 00000001 005d6f11 ...o.........]o. - 3de0 00000000 00007711 00000000 00000100 ......w......... - 3df0 55781100 00000000 00c11100 00000000 Ux.............. - 3e00 0001005d c1110000 00000000 d9110000 ...]............ - 3e10 00000000 010050da 11000000 000000e2 ......P......... - 3e20 11000000 00000001 0050e211 00000000 .........P...... - 3e30 00003a12 00000000 00000100 5d3a1200 ..:.........]:.. - 3e40 00000000 00511200 00000000 00010050 .....Q.........P - 3e50 52120000 00000000 5a120000 00000000 R.......Z....... - 3e60 0100505a 12000000 000000aa 12000000 ..PZ............ - 3e70 00000001 005daa12 00000000 0000c112 .....].......... - 3e80 00000000 00000100 50c21200 00000000 ........P....... - 3e90 00c91200 00000000 00010050 31130000 ...........P1... - 3ea0 00000000 3d130000 00000000 01005d00 ....=.........]. - 3eb0 00000000 00000000 00000000 00000001 ................ - 3ec0 00000000 01010000 00000000 00391000 .............9.. - 3ed0 00000000 007b1000 00000000 0004000a .....{.......... - 3ee0 00049fc7 10000000 000000de 10000000 ................ - 3ef0 00000001 005fde10 00000000 0000e610 ....._.......... - 3f00 00000000 00000400 7f80789f e6100000 ..........x..... - 3f10 00000000 f1100000 00000000 01005f16 .............._. - 3f20 11000000 0000001b 11000000 00000001 ................ - 3f30 00501b11 00000000 00003011 00000000 .P........0..... - 3f40 00000300 91807f31 13000000 0000003d .......1.......= - 3f50 13000000 00000004 000a0004 9f000000 ................ - 3f60 00000000 00000000 00000000 00020000 ................ - 3f70 00000039 10000000 0000007b 10000000 ...9.......{.... - 3f80 00000004 000a0004 9fd41000 00000000 ................ - 3f90 00f11000 00000000 0001005e 31130000 ...........^1... - 3fa0 00000000 3d130000 00000000 04000a00 ....=........... - 3fb0 049f0000 00000000 00000000 00000000 ................ - 3fc0 00000000 00000000 00000000 00000000 ................ - 3fd0 41100000 00000000 4c100000 00000000 A.......L....... - 3fe0 0100504c 10000000 000000c4 10000000 ..PL............ - 3ff0 00000001 005cc410 00000000 0000e510 .....\.......... - 4000 00000000 00000100 50e61000 00000000 ........P....... - 4010 00f11000 00000000 00010050 f1100000 ...........P.... - 4020 00000000 6d110000 00000000 01005c78 ....m.........\x - 4030 11000000 000000c9 12000000 00000001 ................ - 4040 005c3113 00000000 00003d13 00000000 .\1.......=..... - 4050 00000100 50000000 00000000 00000000 ....P........... - 4060 00000000 00000000 00000000 00000000 ................ - 4070 00991100 00000000 009d1100 00000000 ................ - 4080 00010050 9d110000 00000000 0b120000 ...P............ - 4090 00000000 01005f28 12000000 0000002b ......_(.......+ - 40a0 12000000 00000001 00502b12 00000000 .........P+..... - 40b0 00007d12 00000000 00000100 5f981200 ..}........._... - 40c0 00000000 009b1200 00000000 00010050 ...............P - 40d0 9b120000 00000000 c9120000 00000000 ................ - 40e0 01005f00 00000000 00000000 00000000 .._............. - 40f0 00000000 00000001 000100af 11000000 ................ - 4100 000000b2 11000000 00000001 0050b211 .............P.. - 4110 00000000 00000b12 00000000 00000300 ................ - 4120 91a87f28 12000000 0000007d 12000000 ...(.......}.... - 4130 00000002 00389f98 12000000 000000c9 .....8.......... - 4140 12000000 00000002 00389f00 00000000 .........8...... - 4150 00000000 00000000 000000 ........... -Contents of section .debug_aranges: - 0000 2c000000 02000000 00000800 00000000 ,............... - 0010 00000000 00000000 61130000 00000000 ........a....... - 0020 00000000 00000000 00000000 00000000 ................ -Contents of section .debug_ranges: - 0000 4b000000 00000000 b5000000 00000000 K............... - 0010 c8000000 00000000 d8000000 00000000 ................ - 0020 00000000 00000000 00000000 00000000 ................ - 0030 50000000 00000000 70000000 00000000 P.......p....... - 0040 81000000 00000000 b5000000 00000000 ................ - 0050 c8000000 00000000 d8000000 00000000 ................ - 0060 00000000 00000000 00000000 00000000 ................ - 0070 5a010000 00000000 5a010000 00000000 Z.......Z....... - 0080 5c010000 00000000 a1010000 00000000 \............... - 0090 a2010000 00000000 a5010000 00000000 ................ - 00a0 a8010000 00000000 b1010000 00000000 ................ - 00b0 00000000 00000000 00000000 00000000 ................ - 00c0 5a010000 00000000 5a010000 00000000 Z.......Z....... - 00d0 5c010000 00000000 a1010000 00000000 \............... - 00e0 00000000 00000000 00000000 00000000 ................ - 00f0 52030000 00000000 5c030000 00000000 R.......\....... - 0100 6b030000 00000000 c0030000 00000000 k............... - 0110 00000000 00000000 00000000 00000000 ................ - 0120 8e030000 00000000 92030000 00000000 ................ - 0130 96030000 00000000 9c030000 00000000 ................ - 0140 a0030000 00000000 c0030000 00000000 ................ - 0150 00000000 00000000 00000000 00000000 ................ - 0160 c0030000 00000000 07040000 00000000 ................ - 0170 07040000 00000000 68040000 00000000 ........h....... - 0180 68040000 00000000 87040000 00000000 h............... - 0190 87040000 00000000 9c040000 00000000 ................ - 01a0 9c040000 00000000 ae040000 00000000 ................ - 01b0 ae040000 00000000 44050000 00000000 ........D....... - 01c0 44050000 00000000 b0050000 00000000 D............... - 01d0 b0050000 00000000 cc050000 00000000 ................ - 01e0 cc050000 00000000 de050000 00000000 ................ - 01f0 de050000 00000000 f0050000 00000000 ................ - 0200 f0050000 00000000 67060000 00000000 ........g....... - 0210 98070000 00000000 c5070000 00000000 ................ - 0220 c5070000 00000000 20080000 00000000 ........ ....... - 0230 20080000 00000000 3c080000 00000000 .......<....... - 0240 3c080000 00000000 4e080000 00000000 <.......N....... - 0250 4e080000 00000000 60080000 00000000 N.......`....... - 0260 60080000 00000000 2c090000 00000000 `.......,....... - 0270 2c090000 00000000 80090000 00000000 ,............... - 0280 80090000 00000000 a9090000 00000000 ................ - 0290 a9090000 00000000 c2090000 00000000 ................ - 02a0 c2090000 00000000 c2090000 00000000 ................ - 02b0 c2090000 00000000 5c0a0000 00000000 ........\....... - 02c0 5c0a0000 00000000 890a0000 00000000 \............... - 02d0 890a0000 00000000 a20a0000 00000000 ................ - 02e0 a20a0000 00000000 bb0a0000 00000000 ................ - 02f0 bb0a0000 00000000 080b0000 00000000 ................ - 0300 300b0000 00000000 7b0b0000 00000000 0.......{....... - 0310 7b0b0000 00000000 a50b0000 00000000 {............... - 0320 a50b0000 00000000 be0b0000 00000000 ................ - 0330 be0b0000 00000000 d70b0000 00000000 ................ - 0340 d70b0000 00000000 160c0000 00000000 ................ - 0350 160c0000 00000000 480c0000 00000000 ........H....... - 0360 480c0000 00000000 610c0000 00000000 H.......a....... - 0370 610c0000 00000000 7a0c0000 00000000 a.......z....... - 0380 7a0c0000 00000000 e40c0000 00000000 z............... - 0390 e40c0000 00000000 190d0000 00000000 ................ - 03a0 190d0000 00000000 320d0000 00000000 ........2....... - 03b0 320d0000 00000000 4b0d0000 00000000 2.......K....... - 03c0 4b0d0000 00000000 d20d0000 00000000 K............... - 03d0 00000000 00000000 00000000 00000000 ................ - 03e0 fc030000 00000000 04040000 00000000 ................ - 03f0 07040000 00000000 07040000 00000000 ................ - 0400 07040000 00000000 68040000 00000000 ........h....... - 0410 68040000 00000000 87040000 00000000 h............... - 0420 87040000 00000000 9c040000 00000000 ................ - 0430 9c040000 00000000 ae040000 00000000 ................ - 0440 ae040000 00000000 14050000 00000000 ................ - 0450 98070000 00000000 b2070000 00000000 ................ - 0460 b4070000 00000000 bc070000 00000000 ................ - 0470 400a0000 00000000 5c0a0000 00000000 @.......\....... - 0480 5c0a0000 00000000 890a0000 00000000 \............... - 0490 890a0000 00000000 a20a0000 00000000 ................ - 04a0 a20a0000 00000000 bb0a0000 00000000 ................ - 04b0 bb0a0000 00000000 080b0000 00000000 ................ - 04c0 00000000 00000000 00000000 00000000 ................ - 04d0 fc030000 00000000 04040000 00000000 ................ - 04e0 07040000 00000000 68040000 00000000 ........h....... - 04f0 68040000 00000000 87040000 00000000 h............... - 0500 87040000 00000000 9c040000 00000000 ................ - 0510 9c040000 00000000 ae040000 00000000 ................ - 0520 ae040000 00000000 14050000 00000000 ................ - 0530 98070000 00000000 b2070000 00000000 ................ - 0540 b4070000 00000000 bc070000 00000000 ................ - 0550 400a0000 00000000 5c0a0000 00000000 @.......\....... - 0560 5c0a0000 00000000 890a0000 00000000 \............... - 0570 890a0000 00000000 a20a0000 00000000 ................ - 0580 a20a0000 00000000 bb0a0000 00000000 ................ - 0590 bb0a0000 00000000 080b0000 00000000 ................ - 05a0 00000000 00000000 00000000 00000000 ................ - 05b0 68040000 00000000 68040000 00000000 h.......h....... - 05c0 68040000 00000000 87040000 00000000 h............... - 05d0 87040000 00000000 9c040000 00000000 ................ - 05e0 9c040000 00000000 ae040000 00000000 ................ - 05f0 ae040000 00000000 08050000 00000000 ................ - 0600 00000000 00000000 00000000 00000000 ................ - 0610 68040000 00000000 75040000 00000000 h.......u....... - 0620 7c040000 00000000 87040000 00000000 |............... - 0630 00000000 00000000 00000000 00000000 ................ - 0640 75040000 00000000 7c040000 00000000 u.......|....... - 0650 87040000 00000000 8a040000 00000000 ................ - 0660 91040000 00000000 9c040000 00000000 ................ - 0670 00000000 00000000 00000000 00000000 ................ - 0680 8a040000 00000000 91040000 00000000 ................ - 0690 9c040000 00000000 9e040000 00000000 ................ - 06a0 a5040000 00000000 ae040000 00000000 ................ - 06b0 00000000 00000000 00000000 00000000 ................ - 06c0 9e040000 00000000 a5040000 00000000 ................ - 06d0 ae040000 00000000 bb040000 00000000 ................ - 06e0 00000000 00000000 00000000 00000000 ................ - 06f0 98070000 00000000 98070000 00000000 ................ - 0700 9d070000 00000000 a0070000 00000000 ................ - 0710 a5070000 00000000 ad070000 00000000 ................ - 0720 00000000 00000000 00000000 00000000 ................ - 0730 400a0000 00000000 400a0000 00000000 @.......@....... - 0740 540a0000 00000000 5c0a0000 00000000 T.......\....... - 0750 00000000 00000000 00000000 00000000 ................ - 0760 5c0a0000 00000000 5c0a0000 00000000 \.......\....... - 0770 5c0a0000 00000000 640a0000 00000000 \.......d....... - 0780 670a0000 00000000 6e0a0000 00000000 g.......n....... - 0790 7b0a0000 00000000 810a0000 00000000 {............... - 07a0 850a0000 00000000 890a0000 00000000 ................ - 07b0 890a0000 00000000 a20a0000 00000000 ................ - 07c0 a20a0000 00000000 bb0a0000 00000000 ................ - 07d0 bb0a0000 00000000 d40a0000 00000000 ................ - 07e0 00000000 00000000 00000000 00000000 ................ - 07f0 5c0a0000 00000000 640a0000 00000000 \.......d....... - 0800 670a0000 00000000 6e0a0000 00000000 g.......n....... - 0810 7b0a0000 00000000 810a0000 00000000 {............... - 0820 00000000 00000000 00000000 00000000 ................ - 0830 e60a0000 00000000 ee0a0000 00000000 ................ - 0840 f10a0000 00000000 fa0a0000 00000000 ................ - 0850 00000000 00000000 00000000 00000000 ................ - 0860 3c050000 00000000 41050000 00000000 <.......A....... - 0870 44050000 00000000 44050000 00000000 D.......D....... - 0880 44050000 00000000 b0050000 00000000 D............... - 0890 b0050000 00000000 cc050000 00000000 ................ - 08a0 cc050000 00000000 de050000 00000000 ................ - 08b0 de050000 00000000 f0050000 00000000 ................ - 08c0 f0050000 00000000 58060000 00000000 ........X....... - 08d0 a00c0000 00000000 e40c0000 00000000 ................ - 08e0 e40c0000 00000000 190d0000 00000000 ................ - 08f0 190d0000 00000000 320d0000 00000000 ........2....... - 0900 320d0000 00000000 4b0d0000 00000000 2.......K....... - 0910 4b0d0000 00000000 700d0000 00000000 K.......p....... - 0920 800d0000 00000000 d20d0000 00000000 ................ - 0930 00000000 00000000 00000000 00000000 ................ - 0940 3c050000 00000000 41050000 00000000 <.......A....... - 0950 44050000 00000000 b0050000 00000000 D............... - 0960 b0050000 00000000 cc050000 00000000 ................ - 0970 cc050000 00000000 de050000 00000000 ................ - 0980 de050000 00000000 f0050000 00000000 ................ - 0990 f0050000 00000000 58060000 00000000 ........X....... - 09a0 a00c0000 00000000 e40c0000 00000000 ................ - 09b0 e40c0000 00000000 190d0000 00000000 ................ - 09c0 190d0000 00000000 320d0000 00000000 ........2....... - 09d0 320d0000 00000000 4b0d0000 00000000 2.......K....... - 09e0 4b0d0000 00000000 700d0000 00000000 K.......p....... - 09f0 800d0000 00000000 d20d0000 00000000 ................ - 0a00 00000000 00000000 00000000 00000000 ................ - 0a10 b0050000 00000000 b0050000 00000000 ................ - 0a20 b0050000 00000000 cc050000 00000000 ................ - 0a30 cc050000 00000000 de050000 00000000 ................ - 0a40 de050000 00000000 f0050000 00000000 ................ - 0a50 f0050000 00000000 48060000 00000000 ........H....... - 0a60 00000000 00000000 00000000 00000000 ................ - 0a70 b0050000 00000000 bc050000 00000000 ................ - 0a80 c3050000 00000000 cc050000 00000000 ................ - 0a90 00000000 00000000 00000000 00000000 ................ - 0aa0 bc050000 00000000 c3050000 00000000 ................ - 0ab0 cc050000 00000000 ce050000 00000000 ................ - 0ac0 d5050000 00000000 de050000 00000000 ................ - 0ad0 00000000 00000000 00000000 00000000 ................ - 0ae0 ce050000 00000000 d5050000 00000000 ................ - 0af0 de050000 00000000 e0050000 00000000 ................ - 0b00 e7050000 00000000 f0050000 00000000 ................ - 0b10 00000000 00000000 00000000 00000000 ................ - 0b20 e0050000 00000000 e7050000 00000000 ................ - 0b30 f0050000 00000000 fd050000 00000000 ................ - 0b40 00000000 00000000 00000000 00000000 ................ - 0b50 a00c0000 00000000 a50c0000 00000000 ................ - 0b60 a80c0000 00000000 ae0c0000 00000000 ................ - 0b70 b10c0000 00000000 ba0c0000 00000000 ................ - 0b80 00000000 00000000 00000000 00000000 ................ - 0b90 c80c0000 00000000 c80c0000 00000000 ................ - 0ba0 dc0c0000 00000000 e40c0000 00000000 ................ - 0bb0 00000000 00000000 00000000 00000000 ................ - 0bc0 e40c0000 00000000 e40c0000 00000000 ................ - 0bd0 e40c0000 00000000 f00c0000 00000000 ................ - 0be0 f30c0000 00000000 f60c0000 00000000 ................ - 0bf0 070d0000 00000000 0d0d0000 00000000 ................ - 0c00 150d0000 00000000 190d0000 00000000 ................ - 0c10 190d0000 00000000 320d0000 00000000 ........2....... - 0c20 320d0000 00000000 4b0d0000 00000000 2.......K....... - 0c30 4b0d0000 00000000 640d0000 00000000 K.......d....... - 0c40 00000000 00000000 00000000 00000000 ................ - 0c50 e40c0000 00000000 f00c0000 00000000 ................ - 0c60 f30c0000 00000000 f60c0000 00000000 ................ - 0c70 070d0000 00000000 0d0d0000 00000000 ................ - 0c80 00000000 00000000 00000000 00000000 ................ - 0c90 800d0000 00000000 800d0000 00000000 ................ - 0ca0 850d0000 00000000 8d0d0000 00000000 ................ - 0cb0 920d0000 00000000 9a0d0000 00000000 ................ - 0cc0 00000000 00000000 00000000 00000000 ................ - 0cd0 c5070000 00000000 c5070000 00000000 ................ - 0ce0 c5070000 00000000 20080000 00000000 ........ ....... - 0cf0 20080000 00000000 3c080000 00000000 .......<....... - 0d00 3c080000 00000000 4e080000 00000000 <.......N....... - 0d10 4e080000 00000000 60080000 00000000 N.......`....... - 0d20 60080000 00000000 f0080000 00000000 `............... - 0d30 f9080000 00000000 20090000 00000000 ........ ....... - 0d40 4f0b0000 00000000 600b0000 00000000 O.......`....... - 0d50 f80b0000 00000000 160c0000 00000000 ................ - 0d60 160c0000 00000000 480c0000 00000000 ........H....... - 0d70 480c0000 00000000 610c0000 00000000 H.......a....... - 0d80 610c0000 00000000 7a0c0000 00000000 a.......z....... - 0d90 7a0c0000 00000000 a00c0000 00000000 z............... - 0da0 700d0000 00000000 800d0000 00000000 p............... - 0db0 00000000 00000000 00000000 00000000 ................ - 0dc0 20080000 00000000 20080000 00000000 ....... ....... - 0dd0 20080000 00000000 3c080000 00000000 .......<....... - 0de0 3c080000 00000000 4e080000 00000000 <.......N....... - 0df0 4e080000 00000000 60080000 00000000 N.......`....... - 0e00 60080000 00000000 bf080000 00000000 `............... - 0e10 00000000 00000000 00000000 00000000 ................ - 0e20 20080000 00000000 2c080000 00000000 .......,....... - 0e30 33080000 00000000 3c080000 00000000 3.......<....... - 0e40 00000000 00000000 00000000 00000000 ................ - 0e50 2c080000 00000000 33080000 00000000 ,.......3....... - 0e60 3c080000 00000000 3e080000 00000000 <.......>....... - 0e70 45080000 00000000 4e080000 00000000 E.......N....... - 0e80 00000000 00000000 00000000 00000000 ................ - 0e90 3e080000 00000000 45080000 00000000 >.......E....... - 0ea0 4e080000 00000000 50080000 00000000 N.......P....... - 0eb0 57080000 00000000 60080000 00000000 W.......`....... - 0ec0 00000000 00000000 00000000 00000000 ................ - 0ed0 50080000 00000000 57080000 00000000 P.......W....... - 0ee0 60080000 00000000 6d080000 00000000 `.......m....... - 0ef0 00000000 00000000 00000000 00000000 ................ - 0f00 cb080000 00000000 d0080000 00000000 ................ - 0f10 d3080000 00000000 d6080000 00000000 ................ - 0f20 d9080000 00000000 dd080000 00000000 ................ - 0f30 e0080000 00000000 e5080000 00000000 ................ - 0f40 00000000 00000000 00000000 00000000 ................ - 0f50 f80b0000 00000000 f80b0000 00000000 ................ - 0f60 0e0c0000 00000000 160c0000 00000000 ................ - 0f70 00000000 00000000 00000000 00000000 ................ - 0f80 160c0000 00000000 160c0000 00000000 ................ - 0f90 160c0000 00000000 160c0000 00000000 ................ - 0fa0 270c0000 00000000 330c0000 00000000 '.......3....... - 0fb0 360c0000 00000000 480c0000 00000000 6.......H....... - 0fc0 480c0000 00000000 610c0000 00000000 H.......a....... - 0fd0 610c0000 00000000 7a0c0000 00000000 a.......z....... - 0fe0 7a0c0000 00000000 930c0000 00000000 z............... - 0ff0 00000000 00000000 00000000 00000000 ................ - 1000 160c0000 00000000 160c0000 00000000 ................ - 1010 270c0000 00000000 330c0000 00000000 '.......3....... - 1020 360c0000 00000000 440c0000 00000000 6.......D....... - 1030 00000000 00000000 00000000 00000000 ................ - 1040 f0080000 00000000 f9080000 00000000 ................ - 1050 20090000 00000000 2c090000 00000000 .......,....... - 1060 2c090000 00000000 80090000 00000000 ,............... - 1070 80090000 00000000 a9090000 00000000 ................ - 1080 a9090000 00000000 c2090000 00000000 ................ - 1090 c2090000 00000000 c2090000 00000000 ................ - 10a0 c2090000 00000000 400a0000 00000000 ........@....... - 10b0 300b0000 00000000 4f0b0000 00000000 0.......O....... - 10c0 600b0000 00000000 7b0b0000 00000000 `.......{....... - 10d0 7b0b0000 00000000 a50b0000 00000000 {............... - 10e0 a50b0000 00000000 be0b0000 00000000 ................ - 10f0 be0b0000 00000000 d70b0000 00000000 ................ - 1100 d70b0000 00000000 f80b0000 00000000 ................ - 1110 00000000 00000000 00000000 00000000 ................ - 1120 80090000 00000000 80090000 00000000 ................ - 1130 80090000 00000000 a9090000 00000000 ................ - 1140 a9090000 00000000 c2090000 00000000 ................ - 1150 c2090000 00000000 c2090000 00000000 ................ - 1160 c2090000 00000000 1e0a0000 00000000 ................ - 1170 00000000 00000000 00000000 00000000 ................ - 1180 80090000 00000000 8a090000 00000000 ................ - 1190 91090000 00000000 93090000 00000000 ................ - 11a0 9d090000 00000000 a0090000 00000000 ................ - 11b0 a3090000 00000000 a9090000 00000000 ................ - 11c0 00000000 00000000 00000000 00000000 ................ - 11d0 8a090000 00000000 91090000 00000000 ................ - 11e0 9a090000 00000000 9d090000 00000000 ................ - 11f0 a0090000 00000000 a3090000 00000000 ................ - 1200 b5090000 00000000 bc090000 00000000 ................ - 1210 c2090000 00000000 c2090000 00000000 ................ - 1220 00000000 00000000 00000000 00000000 ................ - 1230 93090000 00000000 9a090000 00000000 ................ - 1240 a9090000 00000000 ab090000 00000000 ................ - 1250 b2090000 00000000 b5090000 00000000 ................ - 1260 bc090000 00000000 c2090000 00000000 ................ - 1270 00000000 00000000 00000000 00000000 ................ - 1280 ab090000 00000000 b2090000 00000000 ................ - 1290 c2090000 00000000 cf090000 00000000 ................ - 12a0 00000000 00000000 00000000 00000000 ................ - 12b0 300b0000 00000000 350b0000 00000000 0.......5....... - 12c0 380b0000 00000000 3e0b0000 00000000 8.......>....... - 12d0 410b0000 00000000 4a0b0000 00000000 A.......J....... - 12e0 00000000 00000000 00000000 00000000 ................ - 12f0 600b0000 00000000 600b0000 00000000 `.......`....... - 1300 710b0000 00000000 740b0000 00000000 q.......t....... - 1310 760b0000 00000000 7b0b0000 00000000 v.......{....... - 1320 00000000 00000000 00000000 00000000 ................ - 1330 7b0b0000 00000000 7b0b0000 00000000 {.......{....... - 1340 7b0b0000 00000000 830b0000 00000000 {............... - 1350 850b0000 00000000 8c0b0000 00000000 ................ - 1360 980b0000 00000000 9e0b0000 00000000 ................ - 1370 a10b0000 00000000 a50b0000 00000000 ................ - 1380 a50b0000 00000000 be0b0000 00000000 ................ - 1390 be0b0000 00000000 d70b0000 00000000 ................ - 13a0 d70b0000 00000000 f00b0000 00000000 ................ - 13b0 00000000 00000000 00000000 00000000 ................ - 13c0 7b0b0000 00000000 830b0000 00000000 {............... - 13d0 850b0000 00000000 8c0b0000 00000000 ................ - 13e0 980b0000 00000000 9e0b0000 00000000 ................ - 13f0 00000000 00000000 00000000 00000000 ................ - 1400 67060000 00000000 5b070000 00000000 g.......[....... - 1410 62070000 00000000 65070000 00000000 b.......e....... - 1420 67070000 00000000 6a070000 00000000 g.......j....... - 1430 6e070000 00000000 75070000 00000000 n.......u....... - 1440 79070000 00000000 90070000 00000000 y............... - 1450 080b0000 00000000 300b0000 00000000 ........0....... - 1460 d20d0000 00000000 ea0d0000 00000000 ................ - 1470 00000000 00000000 00000000 00000000 ................ - 1480 7d060000 00000000 5b070000 00000000 }.......[....... - 1490 62070000 00000000 65070000 00000000 b.......e....... - 14a0 67070000 00000000 6a070000 00000000 g.......j....... - 14b0 6e070000 00000000 75070000 00000000 n.......u....... - 14c0 79070000 00000000 90070000 00000000 y............... - 14d0 00000000 00000000 00000000 00000000 ................ - 14e0 92060000 00000000 92060000 00000000 ................ - 14f0 b2060000 00000000 b2060000 00000000 ................ - 1500 d2060000 00000000 d2060000 00000000 ................ - 1510 00000000 00000000 00000000 00000000 ................ - 1520 58070000 00000000 5b070000 00000000 X.......[....... - 1530 62070000 00000000 65070000 00000000 b.......e....... - 1540 67070000 00000000 6a070000 00000000 g.......j....... - 1550 6e070000 00000000 75070000 00000000 n.......u....... - 1560 79070000 00000000 90070000 00000000 y............... - 1570 00000000 00000000 00000000 00000000 ................ - 1580 fe0e0000 00000000 fe0e0000 00000000 ................ - 1590 000f0000 00000000 1a0f0000 00000000 ................ - 15a0 00000000 00000000 00000000 00000000 ................ - 15b0 24100000 00000000 62110000 00000000 $.......b....... - 15c0 66110000 00000000 69110000 00000000 f.......i....... - 15d0 73110000 00000000 c9120000 00000000 s............... - 15e0 31130000 00000000 3d130000 00000000 1.......=....... - 15f0 00000000 00000000 00000000 00000000 ................ - 1600 24100000 00000000 2c100000 00000000 $.......,....... - 1610 2c100000 00000000 62110000 00000000 ,.......b....... - 1620 66110000 00000000 69110000 00000000 f.......i....... - 1630 73110000 00000000 c9120000 00000000 s............... - 1640 31130000 00000000 3d130000 00000000 1.......=....... - 1650 00000000 00000000 00000000 00000000 ................ -Contents of section .debug_line: - 0000 18190000 0200ce01 00000101 fb0e0d00 ................ - 0010 01010101 00000001 0000012f 686f6d65 .........../home - 0020 2f636865 6e7a697a 68616e2f 68656176 /chenzizhan/heav - 0030 796b6565 70657263 7a7a2f46 69656c64 ykeeperczz/Field - 0040 53746174 2f737263 2f746167 73002f68 Stat/src/tags./h - 0050 6f6d652f 6368656e 7a697a68 616e2f68 ome/chenzizhan/h - 0060 65617679 6b656570 6572637a 7a2f4669 eavykeeperczz/Fi - 0070 656c6453 7461742f 76656e64 6f72732f eldStat/vendors/ - 0080 78786861 7368002f 7573722f 6c69622f xxhash./usr/lib/ - 0090 6763632f 7838365f 36342d72 65646861 gcc/x86_64-redha - 00a0 742d6c69 6e75782f 382f696e 636c7564 t-linux/8/includ - 00b0 65002f75 73722f69 6e636c75 64652f62 e./usr/include/b - 00c0 69747300 2f757372 2f696e63 6c756465 its./usr/include - 00d0 2f626974 732f7479 70657300 2f757372 /bits/types./usr - 00e0 2f696e63 6c756465 002f686f 6d652f63 /include./home/c - 00f0 68656e7a 697a6861 6e2f6865 6176796b henzizhan/heavyk - 0100 65657065 72637a7a 2f466965 6c645374 eeperczz/FieldSt - 0110 61742f69 6e636c75 64652f66 69656c64 at/include/field - 0120 73746174 00006d79 5f75745f 68617368 stat..my_ut_hash - 0130 2e630001 00007878 68617368 2e680002 .c....xxhash.h.. - 0140 00007374 64646566 2e680003 00007479 ..stddef.h....ty - 0150 7065732e 68000400 00737472 7563745f pes.h....struct_ - 0160 46494c45 2e680005 00004649 4c452e68 FILE.h....FILE.h - 0170 00050000 73746469 6f2e6800 06000073 ....stdio.h....s - 0180 79735f65 72726c69 73742e68 00040000 ys_errlist.h.... - 0190 73746469 6e742d75 696e746e 2e680004 stdint-uintn.h.. - 01a0 00006669 656c6473 7461742e 68000700 ..fieldstat.h... - 01b0 00737464 6c69622e 68000600 00737472 .stdlib.h....str - 01c0 696e672e 68000600 003c6275 696c742d ing.h....<built- - 01d0 696e3e00 00000000 05010009 02000000 in>............. - 01e0 00000000 00031701 05051305 17060105 ................ - 01f0 013d06cb 05051305 01061105 0a830508 .=.............. - 0200 82051a00 02040158 05010326 9e050506 .......X...&.... - 0210 035e0812 05080601 0511af05 09030958 .^.............X - 0220 05110603 0c580515 06010514 c8d60529 .....X.........) - 0230 00020402 06036b01 052a0002 04020601 ......k..*...... - 0240 05180002 0402064a 00020402 064a0505 .......J.....J.. - 0250 00020402 4a050906 59131305 0d060105 ....J...Y....... - 0260 0ce40509 064d0512 0601050c 3c050906 .....M......<... - 0270 5b051192 05140601 03799e05 01031b66 [........y.....f - 0280 05110603 71082e05 14060105 01060316 ....q........... - 0290 08740505 13050106 110511bb 3c050f58 .t..........<..X - 02a0 0505064b 05150601 05103c05 05063d05 ...K......<...=. - 02b0 0f060105 053c0506 06d2050d 030d0105 .....<.......... - 02c0 1f060105 1d90050d 064b0601 05011758 .........K.....X - 02d0 20050d06 03776605 25060105 2058050d ....wf.%... X.. - 02e0 06590501 061a5820 050d0603 74820527 .Y....X ....t..' - 02f0 06010522 4a050d06 4b050106 030b0158 ..."J...K......X - 0300 20063105 05130508 06010501 57051506 .1.........W... - 0310 5c050106 0e050532 05260608 ac051501 \......2.&...... - 0320 0505064a 05090659 1383050c 0601050d ...J...Y........ - 0330 06670682 05260654 05150105 05060158 .g...&.T.......X - 0340 06190501 06130505 1f05013d 05053b82 ...........=..;. - 0350 050106f6 05051313 13050106 0f0513bd ................ - 0360 3c3c0505 06590508 06010505 064d050b <<...Y.......M.. - 0370 06010517 3c05083c 0505064e 05010603 ....<..<...N.... - 0380 0ad65820 05090603 73740501 06030d01 ..X ....st...... - 0390 05190373 4a2e0501 030d2e20 20050d06 ...sJ...... ... - 03a0 62051406 01050186 5805141c 050d069a b.......X....... - 03b0 05270601 4a050d06 bc052506 01d60501 .'..J.....%..... - 03c0 0603109e 05051305 31060105 29740501 ........1...)t.. - 03d0 75063105 05130512 06010505 062f050a u.1........../.. - 03e0 01051801 05050601 05090002 040306d7 ................ - 03f0 051c0002 04030601 050f0002 04037405 ..............t. - 0400 09000204 03062f05 0f000204 03060105 ....../......... - 0410 25000204 03066405 18000204 03010505 %.....d......... - 0420 00020403 06010002 04035806 16051706 ..........X..... - 0430 0105012f 06a10505 13051606 0105012f .../.........../ - 0440 06d90505 13130501 06100505 8405012c ..............., - 0450 05058482 065a050a 01051501 05050601 .....Z.......... - 0460 90050d06 0309f2f3 05200002 04020376 ......... .....v - 0470 01051500 02040201 05050002 0402064a ...............J - 0480 0509065a 08910518 06010509 3c050d06 ...Z........<... - 0490 03099e08 bb052003 73010515 01050506 ...... .s....... - 04a0 01580603 1401050c 06010501 8306d905 .X.............. - 04b0 05131305 01061004 02050503 b40f2e05 ................ - 04c0 274b0401 050103cb 70660402 051e0603 'K......pf...... - 04d0 b10fe405 05141305 27069f05 174b0510 ........'....K.. - 04e0 6705174b 05056206 5913050c 0603d67a g..K..b.Y......z - 04f0 01051703 aa054a05 05064b13 050c0603 ......J...K..... - 0500 d47a0105 1703ac05 66050506 4c050e03 .z......f...L... - 0510 d07a0105 0514050c 060108e4 05050603 .z.............. - 0520 af050106 01040105 150603c9 70010505 ............p... - 0530 06010228 12050906 84040205 1d0603c5 ...(............ - 0540 0fba0536 4b040105 0903ba70 4a040205 ...6K......pJ... - 0550 010603bb 0f3c0505 14050917 13140536 .....<.........6 - 0560 06133c05 1d490509 06590536 0601051d ..<..I...Y.6.... - 0570 58051a3c 05090668 05120601 051c3c05 X..<...h......<. - 0580 0c4a0509 06a4050c 0601051d 0375ba05 .J...........u.. - 0590 09060317 4a051606 01050c4a 90050d06 ....J......J.... - 05a0 08850511 13060105 0506039a 7d0103de ............}... - 05b0 7e011303 af7e0103 d2010105 1003de01 ~....~.......... - 05c0 01050514 05120601 05099e05 05063d05 ..............=. - 05d0 12061105 0a750505 064b0509 06010505 .....u...K...... - 05e0 06032574 0601054a 0603ff01 01051113 ..%t...J........ - 05f0 06010505 0603997d 0103de7e 011303af .......}...~.... - 0600 7e0103d2 01010510 03de0101 05051405 ~............... - 0610 09060105 05063d05 12061105 0a750505 ......=......u.. - 0620 064b0509 06010505 06032574 0601054a .K........%t...J - 0630 06038002 01051113 06010505 0603987d ...............} - 0640 0103de7e 011303af 7e0103d2 01010510 ...~....~....... - 0650 03de0101 05051405 09060105 05062f05 ............../. - 0660 12061105 0a750505 063d0509 06010505 .....u...=...... - 0670 06032566 0601054a 06038102 01051113 ..%f...J........ - 0680 06010505 0603977d 0103de7e 011303af .......}...~.... - 0690 7e0103d2 01010510 03de0101 05051405 ~............... - 06a0 09060105 05063d05 0a060105 05063d05 ......=.......=. - 06b0 09060105 05060325 74060105 4a060382 .......%t...J... - 06c0 02010515 13050d06 01024d12 05090616 ..........M..... - 06d0 050c0601 ba040105 0906039b 70010517 ............p... - 06e0 06010509 2e050d06 03090890 0402051d ................ - 06f0 0603bb0f ba040105 0d03c570 58040205 ...........pX... - 0700 010603b1 0f3c0505 14050917 13051d06 .....<.......... - 0710 01050906 4c051d06 0105363d 74051d49 ....L.....6=t..I - 0720 05090659 05360601 051d7405 1a3c0509 ...Y.6....t..<.. - 0730 06a00512 0601051c 3c050c4a 050906a4 ........<..J.... - 0740 050c0601 05090603 0c900516 0601050c ................ - 0750 4a90050d 06089305 11130601 05050603 J............... - 0760 9a7d0103 de7e0113 03af7e01 03d20101 .}...~....~..... - 0770 051003de 01010505 14051206 0105099e ................ - 0780 0505062f 05120611 050a7505 05063d05 .../......u...=. - 0790 09060105 05060325 66060105 4a0603ff .......%f...J... - 07a0 01010511 13060105 05060399 7d0103de ............}... - 07b0 7e011303 af7e0103 d2010105 1003de01 ~....~.......... - 07c0 01050514 05090601 0505062f 05120611 .........../.... - 07d0 050a7505 05063d05 09060105 05060325 ..u...=........% - 07e0 66060105 4a060380 02010511 13060105 f...J........... - 07f0 05060398 7d0103de 7e011303 af7e0103 ....}...~....~.. - 0800 d2010105 1003de01 01050514 05090601 ................ - 0810 0505062f 05120611 050a7505 05063d05 .../......u...=. - 0820 09060105 05060325 66060105 4a060381 .......%f...J... - 0830 02010511 13060105 05060397 7d0103de ............}... - 0840 7e011303 af7e0103 d2010105 1003de01 ~....~.......... - 0850 01050514 05090601 0505063d 050a0601 ...........=.... - 0860 0505063d 05090601 05050603 25740601 ...=........%t.. - 0870 054a0603 82020105 1513050d 0601024b .J.............K - 0880 12050906 16050c06 01f20401 05200002 ............. .. - 0890 04020603 98700105 15000204 02010505 .....p.......... - 08a0 00020402 064a0603 14ac0402 051d03df .....J.......... - 08b0 0f010505 1414050d 06500508 037a6605 .........P...zf. - 08c0 05060309 82050906 01050506 4c050103 ............L... - 08d0 877e0105 05030d01 05140105 05150509 .~.............. - 08e0 13050f06 13050d57 05090667 050f0105 .......W...g.... - 08f0 0d9f0105 0103e87e 01050514 05091506 .......~........ - 0900 01050d06 03930101 01051106 83050d49 ...............I - 0910 3c4a0674 13050f10 050d6701 050103e8 <J.t......g..... - 0920 7e010505 14050915 0601050d 06039301 ~............... - 0930 01010511 0683050d 493c4a06 7413050f ........I<J.t... - 0940 10050d67 01050103 e87e0105 05140509 ...g.....~...... - 0950 15060105 0d060393 01010106 ac4a0674 .............J.t - 0960 13050f10 06010608 78050d59 010106ba ........x..Y.... - 0970 4a067413 050f1006 4a050d06 2f010106 J.t.....J.../... - 0980 c806ac13 050f1005 0d670101 06c806ac .........g...... - 0990 13050f10 05091605 10034b01 05051405 ..........K..... - 09a0 10060104 01050103 a5723c04 02051003 .........r<..... - 09b0 db0d7404 01050103 a5723c04 02050903 ..t......r<..... - 09c0 db0d2e05 05063d04 01050106 03a47201 ......=.......r. - 09d0 04020509 03dc0d4a 05050675 04010501 .......J...u.... - 09e0 0603a372 01040205 1003dd0d 4a050974 ...r........J..t - 09f0 0505063d 05090601 05050667 05100601 ...=.......g.... - 0a00 05050659 05090611 04010501 03a1722e ...Y..........r. - 0a10 0402050d 0603b60f 82050e03 bb7a0105 .............z.. - 0a20 0514050d 0603c305 01050c03 bd7a5805 .............zX. - 0a30 0d03c305 3c050c03 bd7a5882 050d0603 ....<....zX..... - 0a40 c4050105 1c060104 01051703 b7705804 .............pX. - 0a50 02051c03 c90f2e05 0d068306 01040105 ................ - 0a60 090603b6 7001050d 96040205 0103b40f ....p........... - 0a70 01050514 05091713 14051d06 01053683 ..............6. - 0a80 051d6505 09065905 36060105 1a3c0509 ..e...Y.6....<.. - 0a90 06a00512 0601051c 3c050c4a 050906dc ........<..J.... - 0aa0 050c0601 05090603 0c90050c 0601050d ................ - 0ab0 0608bd05 11130601 05050603 9a7d0103 .............}.. - 0ac0 de7e0113 03af7e01 03d20101 051003de .~....~......... - 0ad0 01010505 14051206 0105099e 0505062f .............../ - 0ae0 05120611 050a7505 05063d05 09060105 ......u...=..... - 0af0 05060325 66060105 4a0603ff 01010511 ...%f...J....... - 0b00 13060105 05060399 7d0103de 7e011303 ........}...~... - 0b10 af7e0103 d2010105 1003de01 01050514 .~.............. - 0b20 05090601 0505062f 05120611 050a7505 ......./......u. - 0b30 05063d05 09060105 05060325 66060105 ..=........%f... - 0b40 4a060380 02010511 13060105 05060398 J............... - 0b50 7d0103de 7e011303 af7e0103 d2010105 }...~....~...... - 0b60 1003de01 01050514 05090601 0505062f .............../ - 0b70 05120611 050a7505 05063d05 09060105 ......u...=..... - 0b80 05060325 66060105 4a060381 02010511 ...%f...J....... - 0b90 13060105 05060397 7d0103de 7e011303 ........}...~... - 0ba0 af7e0103 d2010105 1003de01 01050514 .~.............. - 0bb0 05090601 0505063d 050a0601 0505063d .......=.......= - 0bc0 05090601 05050603 25740601 054a0603 ........%t...J.. - 0bd0 82020105 1513050d 06010820 023c124a ........... .<.J - 0be0 05090616 050c0601 050d06bb 050c0603 ................ - 0bf0 9f7a0105 3603e105 58050c03 9f7a3c05 .z..6...X....z<. - 0c00 3603e105 3c050e06 039d7a3c 05051405 6...<.....z<.... - 0c10 0c060105 1e03e205 4a050c03 9e7a3c58 ........J....z<X - 0c20 050d0603 e2050105 1c0601ac 050d0603 ................ - 0c30 6101050e 03bb7a01 05051405 0c06014a a.....z........J - 0c40 58050d06 03c40501 06010601 051c0601 X............... - 0c50 050d0608 67040103 b970f204 02051d06 ....g....p...... - 0c60 03c10f01 05124d82 05010603 73010505 ......M.....s... - 0c70 14050917 134c051d 0601051c 3f053648 .....L......?.6H - 0c80 051d3b05 09064b05 36060105 1a660509 ..;...K.6....f.. - 0c90 064c0512 0601050c 3c050906 6c050c06 .L......<...l... - 0ca0 01050906 030c8205 0c060105 0d060227 ...............' - 0cb0 15051113 06010505 06039a7d 0103de7e ...........}...~ - 0cc0 011303af 7e0103d2 01010510 03de0101 ....~........... - 0cd0 05051405 1206019e 05097405 05062f05 ..........t.../. - 0ce0 12061105 0974050a 3d050506 3d050a06 .....t..=...=... - 0cf0 1105093d 05050603 25660601 054a0603 ...=....%f...J.. - 0d00 ff010105 11130601 05050603 997d0103 .............}.. - 0d10 de7e0113 03af7e01 03d20101 051003de .~....~......... - 0d20 01010505 14050906 01050506 2f051206 ............/... - 0d30 11050a75 0505063d 05090601 05050603 ...u...=........ - 0d40 25c80601 054a0603 80020105 11130601 %....J.......... - 0d50 05050603 987d0103 de7e0113 03af7e01 .....}...~....~. - 0d60 03d20101 051003de 01010505 14131303 ................ - 0d70 25010601 054a0603 81020105 11130601 %....J.......... - 0d80 05050603 977d0103 de7e0113 03af7e01 .....}...~....~. - 0d90 03d20101 051003de 01010505 14050906 ................ - 0da0 01050506 3d050a06 01050506 3d050906 ....=.......=... - 0db0 01050506 03257406 01054a06 03820201 .....%t...J..... - 0dc0 05151305 0d060108 2e08ac08 2ec80509 ................ - 0dd0 0616050c 06010222 12050d06 03690105 .......".....i.. - 0de0 0e03b57a 01050514 050d0603 c9050105 ...z............ - 0df0 4b58050d 58050c03 b77a9e82 05110603 KX..X....z...... - 0e00 ca050113 06010505 0603a77d 0103de7e ...........}...~ - 0e10 011303af 7e0103d2 01010510 03de0101 ....~........... - 0e20 05051405 12060105 14039e02 82050903 ................ - 0e30 e27d3c05 05064b05 0a060105 05063d03 .}<...K.......=. - 0e40 25010601 051403f7 0101051c 59050903 %...........Y... - 0e50 e37d8205 0f039c02 66051d45 054c064a .}......f..E.L.J - 0e60 05111306 01050506 03a67d01 03de7e01 ..........}...~. - 0e70 1303af7e 0103d201 01051003 de010105 ...~............ - 0e80 05140512 06010509 82050506 4b050a06 ............K... - 0e90 01050506 3d032501 05090603 5b01051d ....=.%.....[... - 0ea0 03980266 054c064a 05111306 01050506 ...f.L.J........ - 0eb0 03a57d01 03de7e01 1303af7e 0103d201 ..}...~....~.... - 0ec0 01051003 de010105 05140512 06010509 ................ - 0ed0 82050506 4b050a06 01050506 3d032501 ....K.......=.%. - 0ee0 05090603 5b01051d 03990266 054c064a ....[......f.L.J - 0ef0 05111306 01050506 03a47d01 03de7e01 ..........}...~. - 0f00 1303af7e 0103d201 01051003 de010105 ...~............ - 0f10 05140512 06010509 82050506 4b050a06 ............K... - 0f20 01050506 3d032501 05090603 5b01051d ....=.%.....[... - 0f30 039a0266 050d064c 130310ba 05360601 ...f...L.....6.. - 0f40 050e0603 9d7a6605 0514050c 06015805 .....zf.......X. - 0f50 1e03e205 3c050c03 9e7a3c90 050d0603 ....<....z<..... - 0f60 e2050105 1c0601d6 05090603 0e01050f ................ - 0f70 06014b4b 3b3b050d 2f2f050f 3d050d71 ..KK;;..//..=..q - 0f80 2e9e0603 7101050c 06039f7a 01053603 ....q......z..6. - 0f90 e1055805 0e06039d 7a3c0505 14050c06 ..X.....z<...... - 0fa0 01051e03 e2056605 0c039e7a 3c90050d ......f....z<... - 0fb0 0603e205 01051c06 01050d06 03635806 .............cX. - 0fc0 08120616 050e03b5 7a010505 14050d06 ........z....... - 0fd0 03c90501 054b5805 0d58050c 03b77a74 .....KX..X....zt - 0fe0 050d03c9 053c050c 03b77a2e 58051106 .....<....z.X... - 0ff0 03ca0501 13060105 050603a7 7d0103de ............}... - 1000 7e011303 af7e0103 d2010105 1003de01 ~....~.......... - 1010 01050514 05120601 0514039e 02820509 ................ - 1020 03e27d2e 0505064b 050a0601 0505063d ..}....K.......= - 1030 03250106 01051403 f7010105 1c4b0509 .%...........K.. - 1040 03e37d82 050f039c 0266051d 37054c06 ..}......f..7.L. - 1050 4a051113 06010505 0603a67d 0103de7e J..........}...~ - 1060 011303af 7e0103d2 01010510 03de0101 ....~........... - 1070 05051405 12060105 09820505 064b050a .............K.. - 1080 06010505 063d0325 01050906 035b0105 .....=.%.....[.. - 1090 1d039802 66054c06 4a051113 06010505 ....f.L.J....... - 10a0 0603a57d 0103de7e 011303af 7e0103d2 ...}...~....~... - 10b0 01010510 03de0101 05051405 12060105 ................ - 10c0 09820505 064b050a 06010505 063d0325 .....K.......=.% - 10d0 01050906 035b0105 1d039902 66054c06 .....[......f.L. - 10e0 4a051113 06010505 0603a47d 0103de7e J..........}...~ - 10f0 011303af 7e0103d2 01010510 03de0101 ....~........... - 1100 05051405 12060105 09820505 064b050a .............K.. - 1110 06010505 063d0325 01050906 035b0105 .....=.%.....[.. - 1120 1d039a02 66050d06 4c130682 06037801 ....f...L.....x. - 1130 050e03b5 7a010505 14050d06 03c90501 ....z........... - 1140 054b5805 0d66050c 03b77aac 82051106 .KX..f....z..... - 1150 03ca0501 13060105 050603a7 7d0103de ............}... - 1160 7e011303 af7e0103 d2010105 1003de01 ~....~.......... - 1170 01050514 05140603 9e020105 1c910512 ................ - 1180 03e17d82 05098205 05064b05 0f06039d ..}.......K..... - 1190 0201050a 03e37d3c 0505063d 03250106 ......}<...=.%.. - 11a0 01050903 5b58051d 03970266 054c064a ....[X.....f.L.J - 11b0 05111306 01050506 03a67d01 03de7e01 ..........}...~. - 11c0 1303af7e 0103d201 01051003 de010105 ...~............ - 11d0 05140512 06010509 82050506 4b050a06 ............K... - 11e0 01050506 3d032501 05090603 5b01051d ....=.%.....[... - 11f0 03980266 054c064a 05111306 01050506 ...f.L.J........ - 1200 03a57d01 03de7e01 1303af7e 0103d201 ..}...~....~.... - 1210 01051003 de010105 05140512 06010509 ................ - 1220 82050506 4b050a06 01050506 3d032501 ....K.......=.%. - 1230 05090603 5b01051d 03990266 054c064a ....[......f.L.J - 1240 05111306 01050506 03a47d01 03de7e01 ..........}...~. - 1250 1303af7e 0103d201 01051003 de010105 ...~............ - 1260 05140512 06010509 82050506 4b050a06 ............K... - 1270 01050506 3d032501 05090603 5b01051d ....=.%.....[... - 1280 039a0266 050d064c 1306c806 03100105 ...f...L........ - 1290 0c06039f 7a010536 03e10558 050e0603 ....z..6...X.... - 12a0 9d7a3c05 0514050c 0601051e 03e20566 .z<............f - 12b0 050c039e 7a3c9005 0d0603e2 0501051c ....z<.......... - 12c0 0601050d 060367d6 050e03b5 7a010505 ......g.....z... - 12d0 14050d06 03c90501 054b5805 0d58050c .........KX..X.. - 12e0 03b77a9e 82051106 03ca0501 13060105 ..z............. - 12f0 050603a7 7d0103de 7e011303 af7e0103 ....}...~....~.. - 1300 d2010105 1003de01 01050514 05120601 ................ - 1310 05098205 05064b05 1406039d 0201050a ......K......... - 1320 03e37d3c 0505063d 03250106 01051403 ..}<...=.%...... - 1330 f7010105 1c910509 03e37d82 050f039c ..........}..... - 1340 02663c05 1d53054c 064a0511 13060105 .f<..S.L.J...... - 1350 050603a6 7d0103de 7e011303 af7e0103 ....}...~....~.. - 1360 d2010105 1003de01 01050514 05120601 ................ - 1370 05098205 05064b05 0a060105 05063d03 ......K.......=. - 1380 25010509 06035b01 051d0398 0266054c %.....[......f.L - 1390 064a0511 13060105 050603a5 7d0103de .J..........}... - 13a0 7e011303 af7e0103 d2010105 1003de01 ~....~.......... - 13b0 01050514 05120601 05098205 05064b05 ..............K. - 13c0 0a060105 05063d03 25010509 06035b01 ......=.%.....[. - 13d0 051d0399 0266054c 064a0511 13060105 .....f.L.J...... - 13e0 050603a4 7d0103de 7e011303 af7e0103 ....}...~....~.. - 13f0 d2010105 1003de01 01050514 05120601 ................ - 1400 05098205 05064b05 0a060105 05063d03 ......K.......=. - 1410 25010509 06035b01 051d039a 0266050d %.....[......f.. - 1420 064c1306 ba060372 01050e03 bb7a0105 .L.....r.....z.. - 1430 0514050c 0601f205 0d0603c3 0501050e ................ - 1440 03bb7a01 05051405 0d0603c3 0501050c ..z............. - 1450 03bd7a58 050d03c3 0582050c 03bd7a58 ..zX..........zX - 1460 82050d06 03c40501 051c0601 050d0608 ................ - 1470 9f06ac4a 90040105 050603c7 70580402 ...J........pX.. - 1480 051d03df 0f010505 1414050d 065e0508 .............^.. - 1490 037a6604 01050106 03a17008 2e050513 .zf.......p..... - 14a0 050c0601 74050106 00090200 00000000 ....t........... - 14b0 00000016 05051305 01061105 40910501 ............@... - 14c0 57054021 052c5a05 1b430540 03773c05 W.@!.,[email protected]<. - 14d0 05063e05 2c060158 051c4b05 204b051b ..>.,..X..K. K.. - 14e0 4f051803 793c0505 063d1314 15051b06 O...y<...=...... - 14f0 01051958 0505063e 05010613 4a3c06cb ...X...>....J<.. - 1500 05051305 08060105 05066905 10061305 ..........i..... - 1510 1b490505 064b0508 06010501 99053e52 .I...K........>R - 1520 05010378 3c050506 52053e06 01050103 ...x<...R.>..... - 1530 784a053e 28050103 7866053e 52050506 xJ.>(...xf.>R... - 1540 83050a01 05150105 09000204 03670525 .............g.% - 1550 00020403 06010509 00020403 4a052500 ............J.%. - 1560 02040390 05090002 04037405 2c000204 ..........t.,... - 1570 03065705 15000204 03010505 00020403 ..W............. - 1580 06010669 05130601 05014b66 4a064005 ...i......KfJ.@. - 1590 0513050c 06010505 064b050c 06010505 .........K...... - 15a0 063d0508 06010509 0659050e 06010509 .=.......Y...... - 15b0 06750501 06530547 52050103 784a0547 .u...S.GR...xJ.G - 15c0 28050506 74050106 03780105 47283c05 (...t....x..G(<. - 15d0 0103783c 05475205 0a590547 3b050506 ..x<.GR..Y.G;... - 15e0 3d14050a 01051501 050a0610 05090002 =............... - 15f0 04030631 052e0002 04030601 05090002 ...1............ - 1600 04034a05 2e000204 03580509 00020403 ..J......X...... - 1610 74052400 02040306 57051500 02040301 t.$.....W....... - 1620 05050002 04030601 05015b66 4a06bd05 ..........[fJ... - 1630 05130501 06110537 67050157 05372105 .......7g..W.7!. - 1640 105a0537 3a050506 3d051806 0105104b .Z.7:...=......K - 1650 05135705 05065913 05170601 05050668 ..W...Y........h - 1660 05080601 05090659 05010603 09010513 .......Y........ - 1670 03775805 01030982 2e050906 03792005 .wX..........y . - 1680 2d060105 134a052d 2e051390 0509064b -....J.-.......K - 1690 050e0105 19010658 050d0002 0403064b .......X.......K - 16a0 052a0002 0403064a 00020403 82050d00 .*.....J........ - 16b0 02040374 052c0002 04030681 05190002 ...t.,.......... - 16c0 04030105 09000204 03060105 017a582e .............zX. - 16d0 06af0505 13050106 11050821 05014905 ...........!..I. - 16e0 083d0505 06310501 063d0505 1f050906 .=...1...=...... - 16f0 9c0505bc 0501063d 05051f58 05530600 .......=...X.S.. - 1700 09020000 00000000 00001605 05140553 ...............S - 1710 0610050c 304a3c58 05010600 09020000 ....0J<X........ - 1720 00000000 00001705 05130501 0611d605 ................ - 1730 089f0522 030cba58 3c050606 03720105 ..."...X<....r.. - 1740 021a1313 05051413 13052206 0105215b .........."...![ - 1750 05225505 05063d13 13052106 01580505 ."U...=...!..X.. - 1760 063d1305 0214050d 01050206 01ac050c .=.............. - 1770 53635805 0646050c 88050306 087c1305 ScX..F.......|.. - 1780 0d060105 03829005 0f060328 08580608 ...........(.X.. - 1790 6666050d 06bb4b05 23060105 1d3d0523 ff....K.#....=.# - 17a0 73050d06 83050f0f 0509b205 2d060105 s...........-... - 17b0 093c052d 66050966 06590519 0601051e .<.-f..f.Y...... - 17c0 06034e08 12050d01 05020658 ac050506 ..N........X.... - 17d0 03359e05 18060105 05590518 65050e3c .5.......Y..e..< - 17e0 05055905 0e3b0505 063d5906 15053239 ..Y..;...=Y...29 - 17f0 0505064b 05130601 05050684 59050106 ...K........Y... - 1800 13050549 05013d4a 2e050549 05110603 ...I..=J...I.... - 1810 61c8051b 06010511 06088305 1b06014a a..............J - 1820 05110608 21052406 0105213d 05243b05 ....!.$...!=.$;. - 1830 11064b05 17010515 e5052706 01051506 ..K.......'..... - 1840 75052c06 01051706 aa051186 f3052506 u.,...........%. - 1850 9e051158 05050659 05110361 9e051b06 ...X...Y...a.... - 1860 01051106 08751305 24060105 213d0524 .....u..$...!=.$ - 1870 3b051106 4b051701 0515d705 27060105 ;...K.......'... - 1880 15067505 2c060105 1706aa06 58051106 ..u.,.......X... - 1890 030f3cf3 0505d705 11037782 051b0601 ..<.......w..... - 18a0 05110608 75130524 06010521 3d05243b ....u..$...!=.$; - 18b0 0511064b 05170105 15d70527 06010515 ...K.......'.... - 18c0 0675052c 06010517 06aa0674 05090603 .u.,.......t.... - 18d0 5b010514 06017405 12580519 5905123b [.....t..X..Y..; - 18e0 0509063d 05190601 0517023a 12050906 ...=.......:.... - 18f0 83050106 03cb0001 05020344 e4740501 ...........D.t.. - 1900 06033f82 05051313 13050106 0f05054d ..?............M - 1910 7406c905 01061302 09000101 t........... -Contents of section .debug_str: - 0000 6f75745f 6b65795f 73697a65 006d656d out_key_size.mem - 0010 50747200 7461675f 68617368 5f6b6579 Ptr.tag_hash_key - 0020 005f7368 6f727462 7566005f 494f5f6c ._shortbuf._IO_l - 0030 6f636b5f 7400696e 70757400 7374726c ock_t.input.strl - 0040 656e0073 74646572 72005f49 4f5f6275 en.stderr._IO_bu - 0050 665f656e 64007661 6c75655f 646f7562 f_end.value_doub - 0060 6c650066 6e765f68 6173685f 63616c63 le.fnv_hash_calc - 0070 756c6174 6f725f69 6e697400 5f494f5f ulator_init._IO_ - 0080 77726974 655f656e 6400756e 7369676e write_end.unsign - 0090 65642069 6e74005f 66726565 7265735f ed int._freeres_ - 00a0 6c697374 005f666c 61677300 58584833 list._flags.XXH3 - 00b0 325f726f 756e6400 6d795f68 6173685f 2_round.my_hash_ - 00c0 6b657900 7461675f 68617368 5f6b6579 key.tag_hash_key - 00d0 5f73746f 7265005f 6d61726b 65727300 _store._markers. - 00e0 6c696d69 7400474e 55204331 3120382e limit.GNU C11 8. - 00f0 352e3020 32303231 30353134 20285265 5.0 20210514 (Re - 0100 64204861 7420382e 352e302d 31382920 d Hat 8.5.0-18) - 0110 2d6d7475 6e653d67 656e6572 6963202d -mtune=generic - - 0120 6d617263 683d7838 362d3634 202d6720 march=x86-64 -g - 0130 2d4f3320 2d737464 3d676e75 3131202d -O3 -std=gnu11 - - 0140 66504943 005f5f62 75696c74 696e5f6d fPIC.__builtin_m - 0150 656d6370 79005858 485f4e41 4d455350 emcpy.XXH_NAMESP - 0160 41434558 58485f45 52524f52 0076616c ACEXXH_ERROR.val - 0170 75655f73 74720074 61675f68 6173685f ue_str.tag_hash_ - 0180 6b65795f 636f6e76 6572745f 746f5f66 key_convert_to_f - 0190 69656c64 73746174 5f746167 0075696e ieldstat_tag.uin - 01a0 7433325f 74007374 646f7574 005f494f t32_t.stdout._IO - 01b0 5f736176 655f656e 64005858 485f4e41 _save_end.XXH_NA - 01c0 4d455350 41434558 58485f65 72726f72 MESPACEXXH_error - 01d0 636f6465 00666c6f 6174005f 494f5f63 code.float._IO_c - 01e0 6f646563 7674006d 616c6c6f 63006669 odecvt.malloc.fi - 01f0 656c6474 61675f6c 6973745f 66726565 eldtag_list_free - 0200 006e5f6f 7574006c 6f6e6720 6c6f6e67 .n_out.long long - 0210 20756e73 69676e65 6420696e 74007379 unsigned int.sy - 0220 735f6572 726c6973 74005f49 4f5f6261 s_errlist._IO_ba - 0230 636b7570 5f626173 65005f6f 66667365 ckup_base._offse - 0240 74007379 735f6e65 7272005f 66696c65 t.sys_nerr._file - 0250 6e6f0058 58485f61 6c69676e 65640066 no.XXH_aligned.f - 0260 6e765f68 6173685f 63616c63 756c6174 nv_hash_calculat - 0270 6f720073 697a655f 7400746f 74616c5f or.size_t.total_ - 0280 6b65795f 73697a65 005f494f 5f726561 key_size._IO_rea - 0290 645f6261 7365005f 426f6f6c 00585848 d_base._Bool.XXH - 02a0 5f4e414d 45535041 43455858 4833325f _NAMESPACEXXH32_ - 02b0 73746174 655f7300 5858485f 4e414d45 state_s.XXH_NAME - 02c0 53504143 45585848 33325f73 74617465 SPACEXXH32_state - 02d0 5f740073 7464696e 00667265 65005858 _t.stdin.free.XX - 02e0 4833325f 66696e61 6c697a65 00585848 H32_finalize.XXH - 02f0 5f494e4c 494e455f 58584833 325f7570 _INLINE_XXH32_up - 0300 64617465 00585848 5f4e414d 45535041 date.XXH_NAMESPA - 0310 43455858 485f4f4b 006c6172 67655f6c CEXXH_OK.large_l - 0320 656e0058 58485f72 6561644c 4533325f en.XXH_readLE32_ - 0330 616c6967 6e005441 475f4353 5452494e align.TAG_CSTRIN - 0340 47006368 6172005f 6d6f6465 00787868 G.char._mode.xxh - 0350 5f753332 00746167 5f686173 685f6b65 _u32.tag_hash_ke - 0360 795f636f 6e737472 7563745f 77697468 y_construct_with - 0370 5f666965 6c647374 61745f74 61670074 _fieldstat_tag.t - 0380 61675f68 6173685f 6b65795f 636f7079 ag_hash_key_copy - 0390 005f494f 5f6d6172 6b657200 5f494f5f ._IO_marker._IO_ - 03a0 72656164 5f707472 006e5f6d 795f7461 read_ptr.n_my_ta - 03b0 67007374 72636d70 0075696e 74385f74 g.strcmp.uint8_t - 03c0 002f686f 6d652f63 68656e7a 697a6861 ./home/chenzizha - 03d0 6e2f6865 6176796b 65657065 72637a7a n/heavykeeperczz - 03e0 2f466965 6c645374 61742f62 75696c64 /FieldStat/build - 03f0 00544147 5f444f55 424c4500 6669656c .TAG_DOUBLE.fiel - 0400 64746167 5f636f70 79005f49 4f5f7772 dtag_copy._IO_wr - 0410 6974655f 62617365 006b6579 5f636d70 ite_base.key_cmp - 0420 006c6f6e 67206c6f 6e672069 6e740058 .long long int.X - 0430 58485f49 4e4c494e 455f5858 4833325f XH_INLINE_XXH32_ - 0440 72657365 74006e5f 74616773 0066735f reset.n_tags.fs_ - 0450 7461675f 74797065 005f494f 5f736176 tag_type._IO_sav - 0460 655f6261 73650072 65745f6e 5f746167 e_base.ret_n_tag - 0470 002f686f 6d652f63 68656e7a 697a6861 ./home/chenzizha - 0480 6e2f6865 6176796b 65657065 72637a7a n/heavykeeperczz - 0490 2f466965 6c645374 61742f73 72632f74 /FieldStat/src/t - 04a0 6167732f 6d795f75 745f6861 73682e63 ags/my_ut_hash.c - 04b0 00666e76 5f686173 685f6361 6c63756c .fnv_hash_calcul - 04c0 61746f72 5f757064 61746500 6b65795f ator_update.key_ - 04d0 6c656e00 76616c75 655f6c6f 6e676c6f len.value_longlo - 04e0 6e670063 616c5f74 61675f68 61736800 ng.cal_tag_hash. - 04f0 73697a65 005f6672 65657265 735f6275 size._freeres_bu - 0500 66007461 675f6200 6d656d73 697a6500 f.tag_b.memsize. - 0510 7461675f 68617368 5f6b6579 5f676574 tag_hash_key_get - 0520 5f636f6d 706f756e 645f6b65 79006d79 _compound_key.my - 0530 5f746167 5f686173 685f6675 6e630066 _tag_hash_func.f - 0540 6e765f68 6173685f 63616c63 756c6174 nv_hash_calculat - 0550 6f725f67 65745f68 61736876 005f5f70 or_get_hashv.__p - 0560 61643500 5858485f 72656164 4c453332 ad5.XXH_readLE32 - 0570 00736565 64006973 5f646565 705f636f .seed.is_deep_co - 0580 70790064 796e616d 69635f6d 656d005f py.dynamic_mem._ - 0590 76746162 6c655f6f 66667365 74007461 vtable_offset.ta - 05a0 675f6861 73685f6b 65795f66 72656500 g_hash_key_free. - 05b0 6e5f7372 63006c6f 6e672064 6f75626c n_src.long doubl - 05c0 65007461 675f6861 73685f6b 65795f63 e.tag_hash_key_c - 05d0 616c5f66 6e765f68 61736800 58584833 al_fnv_hash.XXH3 - 05e0 325f6176 616c616e 63686500 7265745f 2_avalanche.ret_ - 05f0 68617368 5f6b6579 005f494f 5f726561 hash_key._IO_rea - 0600 645f656e 64007368 6f727420 696e7400 d_end.short int. - 0610 6f6e655f 74696d65 5f6b6579 006c6f6e one_time_key.lon - 0620 6720696e 74007878 685f7538 006d795f g int.xxh_u8.my_ - 0630 636d705f 74616700 6669656c 64737461 cmp_tag.fieldsta - 0640 745f7461 67005f49 4f5f7769 64655f64 t_tag._IO_wide_d - 0650 61746100 76616c5f 6c656e00 6275696c ata.val_len.buil - 0660 645f6479 6e616d69 635f6365 6c6c5f6b d_dynamic_cell_k - 0670 65790074 61677300 6f75745f 6b657900 ey.tags.out_key. - 0680 7461675f 68617368 5f6b6579 5f636d70 tag_hash_key_cmp - 0690 005f5f75 696e7438 5f740058 58485f61 .__uint8_t.XXH_a - 06a0 6c69676e 6d656e74 00757365 645f6c65 lignment.used_le - 06b0 6e00696e 69745f6b 65795f73 697a6500 n.init_key_size. - 06c0 7461675f 68617368 5f6b6579 5f63616c tag_hash_key_cal - 06d0 5f686173 685f7661 6c756500 5f776964 _hash_value._wid - 06e0 655f6461 74610058 58485f49 4e4c494e e_data.XXH_INLIN - 06f0 455f5858 4833325f 64696765 7374005f E_XXH32_digest._ - 0700 6c6f636b 006c6f6e 6720756e 7369676e lock.long unsign - 0710 65642069 6e740062 456e6400 5f6f6c64 ed int.bEnd._old - 0720 5f6f6666 73657400 5f494f5f 46494c45 _offset._IO_FILE - 0730 00585848 33325f68 6173685f 74007479 .XXH32_hash_t.ty - 0740 70650075 6e736967 6e656420 63686172 pe.unsigned char - 0750 00544147 5f494e54 45474552 005f5f75 .TAG_INTEGER.__u - 0760 696e7433 325f7400 5f494f5f 77726974 int32_t._IO_writ - 0770 655f7074 72006361 6c63756c 61746f72 e_ptr.calculator - 0780 00746f74 616c5f6c 656e5f33 32006d65 .total_len_32.me - 0790 6d333200 73746174 65006361 6c5f7461 m32.state.cal_ta - 07a0 675f6861 73685f78 78686173 68005f63 g_hash_xxhash._c - 07b0 6f646563 76740058 5848335f 6b536563 odecvt.XXH3_kSec - 07c0 72657400 7265616c 6c6f6300 73746174 ret.realloc.stat - 07d0 65507472 006e5f74 6167005f 5f6f6666 ePtr.n_tag.__off - 07e0 5f740058 58485f6d 656d6370 79007369 _t.XXH_memcpy.si - 07f0 676e6564 20636861 72007265 745f6669 gned char.ret_fi - 0800 656c6473 7461745f 74616700 73747264 eldstat_tag.strd - 0810 75700073 686f7274 20756e73 69676e65 up.short unsigne - 0820 6420696e 74006d65 6d637079 006b6579 d int.memcpy.key - 0830 5f73697a 65007265 73657276 65640072 _size.reserved.r - 0840 656d6169 6e5f6b65 795f7369 7a650064 emain_key_size.d - 0850 6f75626c 65007461 675f6b65 79005f63 ouble.tag_key._c - 0860 6861696e 00585848 5f756e61 6c69676e hain.XXH_unalign - 0870 65640068 61736876 00585848 5f726561 ed.hashv.XXH_rea - 0880 64333200 46494c45 005f666c 61677332 d32.FILE._flags2 - 0890 00616c69 676e006f 6e655f74 696d655f .align.one_time_ - 08a0 6b65795f 73697a65 005f6375 725f636f key_size._cur_co - 08b0 6c756d6e 00646573 74005f5f 6f666636 lumn.dest.__off6 - 08c0 345f7400 5f756e75 73656432 005f494f 4_t._unused2._IO - 08d0 5f627566 5f626173 65007461 675f6100 _buf_base.tag_a. - 08e0 6e65775f 74616773 00 new_tags. -Contents of section .comment: - 0000 00474343 3a202847 4e552920 382e352e .GCC: (GNU) 8.5. - 0010 30203230 32313035 31342028 52656420 0 20210514 (Red - 0020 48617420 382e352e 302d3138 2900 Hat 8.5.0-18). -Contents of section .eh_frame: - 0000 14000000 00000000 017a5200 01781001 .........zR..x.. - 0010 1b0c0708 90010000 10000000 1c000000 ................ - 0020 00000000 04000000 00000000 4c000000 ............L... - 0030 30000000 00000000 c8000000 00420e10 0............B.. - 0040 8e02420e 188d0342 0e208c04 410e2886 ..B....B. ..A.(. - 0050 05410e30 8306580a 0e28440e 20420e18 .A.0..X..(D. B.. - 0060 420e1042 0e08460b 02760a0e 28410e20 B..B..F..v..(A. - 0070 450e1842 0e10420e 08480b00 3c000000 E..B..B..H..<... - 0080 80000000 00000000 6f000000 00410e10 ........o....A.. - 0090 8602440e 18830347 0e20740a 0e18410e ..D....G. t...A. - 00a0 10410e08 460b4e0a 0e18410e 10410e08 .A..F.N...A..A.. - 00b0 480b4c0e 18410e10 410e0800 2c000000 H.L..A..A...,... - 00c0 c0000000 00000000 61000000 00470e10 ........a....G.. - 00d0 8c02440e 18860341 0e208304 0246c30e ..D....A. ...F.. - 00e0 1844c60e 1042cc0e 08000000 40000000 .D...B......@... - 00f0 f0000000 00000000 88000000 00410e10 .............A.. - 0100 8602440e 18830347 0e206b0a 0e18410e ..D....G. k...A. - 0110 10410e08 470b440a 0e18450e 10410e08 .A..G.D...E..A.. - 0120 460b4c0a 0e18410e 10410e08 4a0b0000 F.L...A..A..J... - 0130 10000000 34010000 00000000 0f000000 ....4........... - 0140 00000000 10000000 48010000 00000000 ........H....... - 0150 27000000 00000000 10000000 5c010000 '...........\... - 0160 00000000 03000000 00000000 34000000 ............4... - 0170 70010000 00000000 ad000000 00420e10 p............B.. - 0180 8d02420e 188c0341 0e208604 460e2883 ..B....A. ..F.(. - 0190 05470e40 02940e28 410e2041 0e18420e .G.@...(A. A..B. - 01a0 10420e08 4c000000 a8010000 00000000 .B..L........... - 01b0 9a0a0000 00420e10 8f024c0e 188e0342 .....B....L....B - 01c0 0e208d04 420e288c 05410e30 8606410e . ..B.(..A.0..A. - 01d0 38830747 0ec00103 f7030a0e 38440e30 8..G........8D.0 - 01e0 410e2845 0e20420e 18490e10 420e085f A.(E. B..I..B.._ - 01f0 0b000000 10000000 f8010000 00000000 ................ - 0200 07000000 00000000 28000000 0c020000 ........(....... - 0210 00000000 44000000 00420e10 8c02440e ....D....B....D. - 0220 18860349 0e208304 710e1841 0e10420e ...I. ..q..A..B. - 0230 08000000 34000000 38020000 00000000 ....4...8....... - 0240 6e000000 005a0e10 8d02420e 188c0344 n....Z....B....D - 0250 0e208604 480e2883 054a0e30 750e2841 . ..H.(..J.0u.(A - 0260 0e20410e 18420e10 420e0800 34000000 . A..B..B...4... - 0270 70020000 00000000 65000000 005a0e10 p.......e....Z.. - 0280 8d02420e 188c0345 0e208604 480e2883 ..B....E. ..H.(. - 0290 054a0e30 6b0e2841 0e20410e 18420e10 .J.0k.(A. A..B.. - 02a0 420e0800 34000000 a8020000 00000000 B...4........... - 02b0 86000000 00420e10 8c02410e 18860349 .....B....A....I - 02c0 0e208304 680a0e18 410e104a 0e08410b . ..h...A..J..A. - 02d0 02420e18 410e1042 0e080000 1c000000 .B..A..B........ - 02e0 e0020000 00000000 2d000000 00410e10 ........-....A.. - 02f0 83024d0a 0e084a0b 500e0800 10000000 ..M...J.P....... - 0300 00030000 00000000 0e000000 00000000 ................ - 0310 64000000 14030000 00000000 3d030000 d...........=... - 0320 00420e10 8f02420e 188e0342 0e208d04 .B....B....B. .. - 0330 420e288c 05410e30 8606410e 38830744 B.(..A.0..A.8..D - 0340 0e900103 58010a0e 38440e30 410e2842 ....X...8D.0A.(B - 0350 0e20420e 18420e10 420e084d 0b03a601 . B..B..B..M.... - 0360 0a0e3841 0e30410e 28420e20 420e1842 ..8A.0A.(B. B..B - 0370 0e10420e 08410b00 14000000 7c030000 ..B..A......|... - 0380 00000000 21000000 00440e20 5c0e0800 ....!....D. \... - -Disassembly of section .text: - -0000000000000000 <tag_hash_key_cal_hash_value>: - 0: 8b 47 10 mov 0x10(%rdi),%eax - 3: c3 retq - 4: 66 66 2e 0f 1f 84 00 data16 nopw %cs:0x0(%rax,%rax,1) - b: 00 00 00 00 - f: 90 nop - -0000000000000010 <tag_hash_key_cmp>: - 10: 41 56 push %r14 - 12: 41 55 push %r13 - 14: 41 54 push %r12 - 16: 55 push %rbp - 17: 53 push %rbx - 18: 4c 8b 77 08 mov 0x8(%rdi),%r14 - 1c: 48 8b 46 08 mov 0x8(%rsi),%rax - 20: 4d 85 f6 test %r14,%r14 - 23: 75 1b jne 40 <tag_hash_key_cmp+0x30> - 25: 45 31 ed xor %r13d,%r13d - 28: 48 85 c0 test %rax,%rax - 2b: 41 0f 95 c5 setne %r13b - 2f: 5b pop %rbx - 30: 44 89 e8 mov %r13d,%eax - 33: 5d pop %rbp - 34: 41 5c pop %r12 - 36: 41 5d pop %r13 - 38: 41 5e pop %r14 - 3a: c3 retq - 3b: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) - 40: 49 39 c6 cmp %rax,%r14 - 43: 75 6a jne af <tag_hash_key_cmp+0x9f> - 45: 48 8b 2e mov (%rsi),%rbp - 48: 48 8b 1f mov (%rdi),%rbx - 4b: 45 31 e4 xor %r12d,%r12d - 4e: eb 31 jmp 81 <tag_hash_key_cmp+0x71> - 50: 83 f8 02 cmp $0x2,%eax - 53: 75 1b jne 70 <tag_hash_key_cmp+0x60> - 55: 48 8b 75 10 mov 0x10(%rbp),%rsi - 59: 48 8b 7b 10 mov 0x10(%rbx),%rdi - 5d: e8 00 00 00 00 callq 62 <tag_hash_key_cmp+0x52> - 62: 85 c0 test %eax,%eax - 64: 75 49 jne af <tag_hash_key_cmp+0x9f> - 66: 66 2e 0f 1f 84 00 00 nopw %cs:0x0(%rax,%rax,1) - 6d: 00 00 00 - 70: 49 83 c4 01 add $0x1,%r12 - 74: 48 83 c5 18 add $0x18,%rbp - 78: 48 83 c3 18 add $0x18,%rbx - 7c: 4d 39 e6 cmp %r12,%r14 - 7f: 74 ae je 2f <tag_hash_key_cmp+0x1f> - 81: 48 8b 75 00 mov 0x0(%rbp),%rsi - 85: 48 8b 3b mov (%rbx),%rdi - 88: e8 00 00 00 00 callq 8d <tag_hash_key_cmp+0x7d> - 8d: 41 89 c5 mov %eax,%r13d - 90: 85 c0 test %eax,%eax - 92: 75 1b jne af <tag_hash_key_cmp+0x9f> - 94: 8b 43 08 mov 0x8(%rbx),%eax - 97: 3b 45 08 cmp 0x8(%rbp),%eax - 9a: 75 13 jne af <tag_hash_key_cmp+0x9f> - 9c: 83 f8 01 cmp $0x1,%eax - 9f: 74 27 je c8 <tag_hash_key_cmp+0xb8> - a1: 85 c0 test %eax,%eax - a3: 75 ab jne 50 <tag_hash_key_cmp+0x40> - a5: 48 8b 45 10 mov 0x10(%rbp),%rax - a9: 48 39 43 10 cmp %rax,0x10(%rbx) - ad: 74 c1 je 70 <tag_hash_key_cmp+0x60> - af: 41 bd 01 00 00 00 mov $0x1,%r13d - b5: 5b pop %rbx - b6: 5d pop %rbp - b7: 44 89 e8 mov %r13d,%eax - ba: 41 5c pop %r12 - bc: 41 5d pop %r13 - be: 41 5e pop %r14 - c0: c3 retq - c1: 0f 1f 80 00 00 00 00 nopl 0x0(%rax) - c8: f2 0f 10 43 10 movsd 0x10(%rbx),%xmm0 - cd: 66 0f 2e 45 10 ucomisd 0x10(%rbp),%xmm0 - d2: 7a db jp af <tag_hash_key_cmp+0x9f> - d4: 74 9a je 70 <tag_hash_key_cmp+0x60> - d6: eb d7 jmp af <tag_hash_key_cmp+0x9f> - d8: 0f 1f 84 00 00 00 00 nopl 0x0(%rax,%rax,1) - df: 00 - -00000000000000e0 <fieldtag_copy>: - e0: 55 push %rbp - e1: 48 89 fd mov %rdi,%rbp - e4: 53 push %rbx - e5: 48 89 f3 mov %rsi,%rbx - e8: 48 83 ec 08 sub $0x8,%rsp - ec: 48 8b 3e mov (%rsi),%rdi - ef: e8 00 00 00 00 callq f4 <fieldtag_copy+0x14> - f4: 48 89 45 00 mov %rax,0x0(%rbp) - f8: 8b 43 08 mov 0x8(%rbx),%eax - fb: 89 45 08 mov %eax,0x8(%rbp) - fe: 8b 43 08 mov 0x8(%rbx),%eax - 101: 83 f8 01 cmp $0x1,%eax - 104: 74 22 je 128 <fieldtag_copy+0x48> - 106: 85 c0 test %eax,%eax - 108: 74 36 je 140 <fieldtag_copy+0x60> - 10a: 83 f8 02 cmp $0x2,%eax - 10d: 75 0d jne 11c <fieldtag_copy+0x3c> - 10f: 48 8b 7b 10 mov 0x10(%rbx),%rdi - 113: e8 00 00 00 00 callq 118 <fieldtag_copy+0x38> - 118: 48 89 45 10 mov %rax,0x10(%rbp) - 11c: 48 83 c4 08 add $0x8,%rsp - 120: 5b pop %rbx - 121: 5d pop %rbp - 122: c3 retq - 123: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) - 128: f2 0f 10 43 10 movsd 0x10(%rbx),%xmm0 - 12d: f2 0f 11 45 10 movsd %xmm0,0x10(%rbp) - 132: 48 83 c4 08 add $0x8,%rsp - 136: 5b pop %rbx - 137: 5d pop %rbp - 138: c3 retq - 139: 0f 1f 80 00 00 00 00 nopl 0x0(%rax) - 140: 48 8b 43 10 mov 0x10(%rbx),%rax - 144: 48 89 45 10 mov %rax,0x10(%rbp) - 148: 48 83 c4 08 add $0x8,%rsp - 14c: 5b pop %rbx - 14d: 5d pop %rbp - 14e: c3 retq - 14f: 90 nop - -0000000000000150 <fieldtag_list_free>: - 150: 48 85 ff test %rdi,%rdi - 153: 74 5b je 1b0 <fieldtag_list_free+0x60> - 155: 41 54 push %r12 - 157: 49 89 fc mov %rdi,%r12 - 15a: 55 push %rbp - 15b: 53 push %rbx - 15c: 85 f6 test %esi,%esi - 15e: 7e 41 jle 1a1 <fieldtag_list_free+0x51> - 160: 8d 46 ff lea -0x1(%rsi),%eax - 163: 48 89 fb mov %rdi,%rbx - 166: 48 8d 04 40 lea (%rax,%rax,2),%rax - 16a: 48 8d 6c c7 18 lea 0x18(%rdi,%rax,8),%rbp - 16f: eb 10 jmp 181 <fieldtag_list_free+0x31> - 171: 0f 1f 80 00 00 00 00 nopl 0x0(%rax) - 178: 48 83 c3 18 add $0x18,%rbx - 17c: 48 39 eb cmp %rbp,%rbx - 17f: 74 20 je 1a1 <fieldtag_list_free+0x51> - 181: 48 8b 3b mov (%rbx),%rdi - 184: e8 00 00 00 00 callq 189 <fieldtag_list_free+0x39> - 189: 83 7b 08 02 cmpl $0x2,0x8(%rbx) - 18d: 75 e9 jne 178 <fieldtag_list_free+0x28> - 18f: 48 8b 7b 10 mov 0x10(%rbx),%rdi - 193: 48 83 c3 18 add $0x18,%rbx - 197: e8 00 00 00 00 callq 19c <fieldtag_list_free+0x4c> - 19c: 48 39 eb cmp %rbp,%rbx - 19f: 75 e0 jne 181 <fieldtag_list_free+0x31> - 1a1: 5b pop %rbx - 1a2: 4c 89 e7 mov %r12,%rdi - 1a5: 5d pop %rbp - 1a6: 41 5c pop %r12 - 1a8: e9 00 00 00 00 jmpq 1ad <fieldtag_list_free+0x5d> - 1ad: 0f 1f 00 nopl (%rax) - 1b0: c3 retq - 1b1: 66 66 2e 0f 1f 84 00 data16 nopw %cs:0x0(%rax,%rax,1) - 1b8: 00 00 00 00 - 1bc: 0f 1f 40 00 nopl 0x0(%rax) - -00000000000001c0 <my_cmp_tag>: - 1c0: 55 push %rbp - 1c1: 48 89 f5 mov %rsi,%rbp - 1c4: 53 push %rbx - 1c5: 48 89 fb mov %rdi,%rbx - 1c8: 48 83 ec 08 sub $0x8,%rsp - 1cc: 48 8b 36 mov (%rsi),%rsi - 1cf: 48 8b 3f mov (%rdi),%rdi - 1d2: e8 00 00 00 00 callq 1d7 <my_cmp_tag+0x17> - 1d7: 85 c0 test %eax,%eax - 1d9: 75 18 jne 1f3 <my_cmp_tag+0x33> - 1db: 8b 53 08 mov 0x8(%rbx),%edx - 1de: 8b 4d 08 mov 0x8(%rbp),%ecx - 1e1: 39 ca cmp %ecx,%edx - 1e3: 75 1b jne 200 <my_cmp_tag+0x40> - 1e5: 83 fa 01 cmp $0x1,%edx - 1e8: 74 4e je 238 <my_cmp_tag+0x78> - 1ea: 85 d2 test %edx,%edx - 1ec: 74 3a je 228 <my_cmp_tag+0x68> - 1ee: 83 fa 02 cmp $0x2,%edx - 1f1: 74 1d je 210 <my_cmp_tag+0x50> - 1f3: 48 83 c4 08 add $0x8,%rsp - 1f7: 5b pop %rbx - 1f8: 5d pop %rbp - 1f9: c3 retq - 1fa: 66 0f 1f 44 00 00 nopw 0x0(%rax,%rax,1) - 200: 48 83 c4 08 add $0x8,%rsp - 204: 89 d0 mov %edx,%eax - 206: 29 c8 sub %ecx,%eax - 208: 5b pop %rbx - 209: 5d pop %rbp - 20a: c3 retq - 20b: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) - 210: 48 8b 75 10 mov 0x10(%rbp),%rsi - 214: 48 8b 7b 10 mov 0x10(%rbx),%rdi - 218: 48 83 c4 08 add $0x8,%rsp - 21c: 5b pop %rbx - 21d: 5d pop %rbp - 21e: e9 00 00 00 00 jmpq 223 <my_cmp_tag+0x63> - 223: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) - 228: 48 8b 43 10 mov 0x10(%rbx),%rax - 22c: 2b 45 10 sub 0x10(%rbp),%eax - 22f: eb c2 jmp 1f3 <my_cmp_tag+0x33> - 231: 0f 1f 80 00 00 00 00 nopl 0x0(%rax) - 238: f2 0f 10 43 10 movsd 0x10(%rbx),%xmm0 - 23d: f2 0f 5c 45 10 subsd 0x10(%rbp),%xmm0 - 242: f2 0f 2c c0 cvttsd2si %xmm0,%eax - 246: eb ab jmp 1f3 <my_cmp_tag+0x33> - 248: 0f 1f 84 00 00 00 00 nopl 0x0(%rax,%rax,1) - 24f: 00 - -0000000000000250 <fnv_hash_calculator_init>: - 250: 89 f0 mov %esi,%eax - 252: c1 e0 04 shl $0x4,%eax - 255: 29 f0 sub %esi,%eax - 257: 2d 3b 62 e3 7e sub $0x7ee3623b,%eax - 25c: 89 07 mov %eax,(%rdi) - 25e: c3 retq - 25f: 90 nop - -0000000000000260 <fnv_hash_calculator_update>: - 260: 8b 07 mov (%rdi),%eax - 262: 48 85 d2 test %rdx,%rdx - 265: 74 1d je 284 <fnv_hash_calculator_update+0x24> - 267: 48 01 f2 add %rsi,%rdx - 26a: 66 0f 1f 44 00 00 nopw 0x0(%rax,%rax,1) - 270: 0f be 0e movsbl (%rsi),%ecx - 273: 48 83 c6 01 add $0x1,%rsi - 277: 31 c8 xor %ecx,%eax - 279: 69 c0 93 01 00 01 imul $0x1000193,%eax,%eax - 27f: 48 39 f2 cmp %rsi,%rdx - 282: 75 ec jne 270 <fnv_hash_calculator_update+0x10> - 284: 89 07 mov %eax,(%rdi) - 286: c3 retq - 287: 66 0f 1f 84 00 00 00 nopw 0x0(%rax,%rax,1) - 28e: 00 00 - -0000000000000290 <fnv_hash_calculator_get_hashv>: - 290: 8b 07 mov (%rdi),%eax - 292: c3 retq - 293: 66 66 2e 0f 1f 84 00 data16 nopw %cs:0x0(%rax,%rax,1) - 29a: 00 00 00 00 - 29e: 66 90 xchg %ax,%ax - -00000000000002a0 <cal_tag_hash>: - 2a0: 41 55 push %r13 - 2a2: 41 54 push %r12 - 2a4: 55 push %rbp - 2a5: 48 89 f5 mov %rsi,%rbp - 2a8: 89 d6 mov %edx,%esi - 2aa: 53 push %rbx - 2ab: 48 89 fb mov %rdi,%rbx - 2ae: 48 83 ec 18 sub $0x18,%rsp - 2b2: 4c 8d 64 24 0c lea 0xc(%rsp),%r12 - 2b7: 4c 89 e7 mov %r12,%rdi - 2ba: e8 00 00 00 00 callq 2bf <cal_tag_hash+0x1f> - 2bf: 48 85 ed test %rbp,%rbp - 2c2: 74 76 je 33a <cal_tag_hash+0x9a> - 2c4: 48 83 c3 10 add $0x10,%rbx - 2c8: 48 8d 44 6d 00 lea 0x0(%rbp,%rbp,2),%rax - 2cd: 4c 8d 2c c3 lea (%rbx,%rax,8),%r13 - 2d1: eb 1e jmp 2f1 <cal_tag_hash+0x51> - 2d3: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) - 2d8: ba 08 00 00 00 mov $0x8,%edx - 2dd: 48 89 de mov %rbx,%rsi - 2e0: 4c 89 e7 mov %r12,%rdi - 2e3: e8 00 00 00 00 callq 2e8 <cal_tag_hash+0x48> - 2e8: 48 83 c3 18 add $0x18,%rbx - 2ec: 49 39 dd cmp %rbx,%r13 - 2ef: 74 49 je 33a <cal_tag_hash+0x9a> - 2f1: 48 8b 6b f0 mov -0x10(%rbx),%rbp - 2f5: 48 89 ef mov %rbp,%rdi - 2f8: e8 00 00 00 00 callq 2fd <cal_tag_hash+0x5d> - 2fd: 48 89 ee mov %rbp,%rsi - 300: 4c 89 e7 mov %r12,%rdi - 303: 48 89 c2 mov %rax,%rdx - 306: e8 00 00 00 00 callq 30b <cal_tag_hash+0x6b> - 30b: 8b 43 f8 mov -0x8(%rbx),%eax - 30e: 83 f8 01 cmp $0x1,%eax - 311: 76 c5 jbe 2d8 <cal_tag_hash+0x38> - 313: 83 f8 02 cmp $0x2,%eax - 316: 75 d0 jne 2e8 <cal_tag_hash+0x48> - 318: 48 8b 2b mov (%rbx),%rbp - 31b: 48 83 c3 18 add $0x18,%rbx - 31f: 48 89 ef mov %rbp,%rdi - 322: e8 00 00 00 00 callq 327 <cal_tag_hash+0x87> - 327: 48 89 ee mov %rbp,%rsi - 32a: 4c 89 e7 mov %r12,%rdi - 32d: 48 89 c2 mov %rax,%rdx - 330: e8 00 00 00 00 callq 335 <cal_tag_hash+0x95> - 335: 49 39 dd cmp %rbx,%r13 - 338: 75 b7 jne 2f1 <cal_tag_hash+0x51> - 33a: 4c 89 e7 mov %r12,%rdi - 33d: e8 00 00 00 00 callq 342 <cal_tag_hash+0xa2> - 342: 48 83 c4 18 add $0x18,%rsp - 346: 5b pop %rbx - 347: 5d pop %rbp - 348: 41 5c pop %r12 - 34a: 41 5d pop %r13 - 34c: c3 retq - 34d: 0f 1f 00 nopl (%rax) - -0000000000000350 <cal_tag_hash_xxhash>: - 350: 41 57 push %r15 - 352: 66 0f ef c0 pxor %xmm0,%xmm0 - 356: 8d 82 28 44 23 24 lea 0x24234428(%rdx),%eax - 35c: 41 56 push %r14 - 35e: 41 55 push %r13 - 360: 41 54 push %r12 - 362: 55 push %rbp - 363: 53 push %rbx - 364: 48 81 ec 88 00 00 00 sub $0x88,%rsp - 36b: 0f 29 44 24 50 movaps %xmm0,0x50(%rsp) - 370: 0f 29 44 24 60 movaps %xmm0,0x60(%rsp) - 375: 89 44 24 58 mov %eax,0x58(%rsp) - 379: 8d 82 77 ca eb 85 lea -0x7a143589(%rdx),%eax - 37f: 89 54 24 60 mov %edx,0x60(%rsp) - 383: 81 c2 4f 86 c8 61 add $0x61c8864f,%edx - 389: 0f 29 44 24 70 movaps %xmm0,0x70(%rsp) - 38e: 8b 4c 24 78 mov 0x78(%rsp),%ecx - 392: 89 44 24 5c mov %eax,0x5c(%rsp) - 396: 66 0f 6f 64 24 50 movdqa 0x50(%rsp),%xmm4 - 39c: 89 54 24 64 mov %edx,0x64(%rsp) - 3a0: 66 0f 6f 6c 24 60 movdqa 0x60(%rsp),%xmm5 - 3a6: 41 89 cd mov %ecx,%r13d - 3a9: 0f 29 64 24 20 movaps %xmm4,0x20(%rsp) - 3ae: 0f 29 6c 24 30 movaps %xmm5,0x30(%rsp) - 3b3: 48 c7 44 24 40 00 00 movq $0x0,0x40(%rsp) - 3ba: 00 00 - 3bc: 89 4c 24 48 mov %ecx,0x48(%rsp) - 3c0: 48 85 f6 test %rsi,%rsi - 3c3: 0f 84 f7 09 00 00 je dc0 <cal_tag_hash_xxhash+0xa70> - 3c9: 48 8d 5f 08 lea 0x8(%rdi),%rbx - 3cd: 48 8d 04 76 lea (%rsi,%rsi,2),%rax - 3d1: 44 8b 64 24 20 mov 0x20(%rsp),%r12d - 3d6: 8b 6c 24 24 mov 0x24(%rsp),%ebp - 3da: 48 8d 04 c3 lea (%rbx,%rax,8),%rax - 3de: 48 89 44 24 10 mov %rax,0x10(%rsp) - 3e3: 48 8d 44 24 20 lea 0x20(%rsp),%rax - 3e8: 48 89 44 24 08 mov %rax,0x8(%rsp) - 3ed: 0f 1f 00 nopl (%rax) - 3f0: 4c 8b 7b f8 mov -0x8(%rbx),%r15 - 3f4: 4c 89 ff mov %r15,%rdi - 3f7: e8 00 00 00 00 callq 3fc <cal_tag_hash_xxhash+0xac> - 3fc: 45 8d 0c 04 lea (%r12,%rax,1),%r9d - 400: 48 83 f8 0f cmp $0xf,%rax - 404: 49 89 c6 mov %rax,%r14 - 407: 0f 97 c0 seta %al - 40a: 41 83 f9 0f cmp $0xf,%r9d - 40e: 44 89 4c 24 20 mov %r9d,0x20(%rsp) - 413: 0f 97 c2 seta %dl - 416: 09 d0 or %edx,%eax - 418: 0f b6 c0 movzbl %al,%eax - 41b: 09 e8 or %ebp,%eax - 41d: 89 44 24 24 mov %eax,0x24(%rsp) - 421: 44 89 e8 mov %r13d,%eax - 424: 49 8d 14 06 lea (%r14,%rax,1),%rdx - 428: 48 83 fa 0f cmp $0xf,%rdx - 42c: 0f 86 66 03 00 00 jbe 798 <cal_tag_hash_xxhash+0x448> - 432: 4c 89 fe mov %r15,%rsi - 435: 45 85 ed test %r13d,%r13d - 438: 0f 85 02 06 00 00 jne a40 <cal_tag_hash_xxhash+0x6f0> - 43e: 4b 8d 04 37 lea (%r15,%r14,1),%rax - 442: 4c 8d 48 f0 lea -0x10(%rax),%r9 - 446: 49 39 f1 cmp %rsi,%r9 - 449: 0f 82 b9 00 00 00 jb 508 <cal_tag_hash_xxhash+0x1b8> - 44f: 44 8b 44 24 28 mov 0x28(%rsp),%r8d - 454: 44 8b 54 24 2c mov 0x2c(%rsp),%r10d - 459: 48 89 f1 mov %rsi,%rcx - 45c: 8b 7c 24 30 mov 0x30(%rsp),%edi - 460: 44 8b 5c 24 34 mov 0x34(%rsp),%r11d - 465: 0f 1f 00 nopl (%rax) - 468: 69 11 77 ca eb 85 imul $0x85ebca77,(%rcx),%edx - 46e: 48 83 c1 10 add $0x10,%rcx - 472: 41 01 d0 add %edx,%r8d - 475: 69 51 f4 77 ca eb 85 imul $0x85ebca77,-0xc(%rcx),%edx - 47c: 41 c1 c0 0d rol $0xd,%r8d - 480: 45 69 c0 b1 79 37 9e imul $0x9e3779b1,%r8d,%r8d - 487: 41 01 d2 add %edx,%r10d - 48a: 69 51 f8 77 ca eb 85 imul $0x85ebca77,-0x8(%rcx),%edx - 491: 41 c1 c2 0d rol $0xd,%r10d - 495: 45 69 d2 b1 79 37 9e imul $0x9e3779b1,%r10d,%r10d - 49c: 01 d7 add %edx,%edi - 49e: 69 51 fc 77 ca eb 85 imul $0x85ebca77,-0x4(%rcx),%edx - 4a5: c1 c7 0d rol $0xd,%edi - 4a8: 69 ff b1 79 37 9e imul $0x9e3779b1,%edi,%edi - 4ae: 44 01 da add %r11d,%edx - 4b1: c1 c2 0d rol $0xd,%edx - 4b4: 44 69 da b1 79 37 9e imul $0x9e3779b1,%edx,%r11d - 4bb: 49 39 c9 cmp %rcx,%r9 - 4be: 73 a8 jae 468 <cal_tag_hash_xxhash+0x118> - 4c0: 89 7c 24 18 mov %edi,0x18(%rsp) - 4c4: 66 0f 6e 4c 24 18 movd 0x18(%rsp),%xmm1 - 4ca: 49 29 f1 sub %rsi,%r9 - 4cd: 44 89 5c 24 18 mov %r11d,0x18(%rsp) - 4d2: 66 0f 6e 54 24 18 movd 0x18(%rsp),%xmm2 - 4d8: 49 83 e1 f0 and $0xfffffffffffffff0,%r9 - 4dc: 44 89 44 24 18 mov %r8d,0x18(%rsp) - 4e1: 66 0f 6e 44 24 18 movd 0x18(%rsp),%xmm0 - 4e7: 4a 8d 74 0e 10 lea 0x10(%rsi,%r9,1),%rsi - 4ec: 44 89 54 24 18 mov %r10d,0x18(%rsp) - 4f1: 66 0f 6e 5c 24 18 movd 0x18(%rsp),%xmm3 - 4f7: 66 0f 62 ca punpckldq %xmm2,%xmm1 - 4fb: 66 0f 62 c3 punpckldq %xmm3,%xmm0 - 4ff: 66 0f 6c c1 punpcklqdq %xmm1,%xmm0 - 503: 0f 11 44 24 28 movups %xmm0,0x28(%rsp) - 508: 45 31 ed xor %r13d,%r13d - 50b: 48 39 f0 cmp %rsi,%rax - 50e: 0f 87 cc 05 00 00 ja ae0 <cal_tag_hash_xxhash+0x790> - 514: 8b 03 mov (%rbx),%eax - 516: 83 f8 01 cmp $0x1,%eax - 519: 0f 84 a6 02 00 00 je 7c5 <cal_tag_hash_xxhash+0x475> - 51f: 85 c0 test %eax,%eax - 521: 0f 84 f9 03 00 00 je 920 <cal_tag_hash_xxhash+0x5d0> - 527: 83 f8 02 cmp $0x2,%eax - 52a: 0f 85 e0 03 00 00 jne 910 <cal_tag_hash_xxhash+0x5c0> - 530: 4c 8b 73 08 mov 0x8(%rbx),%r14 - 534: 4c 89 f7 mov %r14,%rdi - 537: e8 00 00 00 00 callq 53c <cal_tag_hash_xxhash+0x1ec> - 53c: 44 8b 64 24 20 mov 0x20(%rsp),%r12d - 541: 49 89 c2 mov %rax,%r10 - 544: 4d 8d 3c 06 lea (%r14,%rax,1),%r15 - 548: 41 01 c4 add %eax,%r12d - 54b: 48 83 f8 0f cmp $0xf,%rax - 54f: 0f 97 c0 seta %al - 552: 41 83 fc 0f cmp $0xf,%r12d - 556: 44 89 64 24 20 mov %r12d,0x20(%rsp) - 55b: 41 0f 97 c0 seta %r8b - 55f: 44 09 c0 or %r8d,%eax - 562: 0f b6 c0 movzbl %al,%eax - 565: 0b 44 24 24 or 0x24(%rsp),%eax - 569: 89 c5 mov %eax,%ebp - 56b: 89 44 24 24 mov %eax,0x24(%rsp) - 56f: 44 89 e8 mov %r13d,%eax - 572: 49 8d 14 02 lea (%r10,%rax,1),%rdx - 576: 48 83 fa 0f cmp $0xf,%rdx - 57a: 0f 86 00 08 00 00 jbe d80 <cal_tag_hash_xxhash+0xa30> - 580: 45 85 ed test %r13d,%r13d - 583: 0f 85 3f 07 00 00 jne cc8 <cal_tag_hash_xxhash+0x978> - 589: 4d 8d 57 f0 lea -0x10(%r15),%r10 - 58d: 4d 39 f2 cmp %r14,%r10 - 590: 0f 82 b2 00 00 00 jb 648 <cal_tag_hash_xxhash+0x2f8> - 596: 8b 54 24 28 mov 0x28(%rsp),%edx - 59a: 8b 7c 24 2c mov 0x2c(%rsp),%edi - 59e: 4c 89 f0 mov %r14,%rax - 5a1: 8b 74 24 30 mov 0x30(%rsp),%esi - 5a5: 44 8b 5c 24 34 mov 0x34(%rsp),%r11d - 5aa: 66 0f 1f 44 00 00 nopw 0x0(%rax,%rax,1) - 5b0: 69 08 77 ca eb 85 imul $0x85ebca77,(%rax),%ecx - 5b6: 48 83 c0 10 add $0x10,%rax - 5ba: 01 ca add %ecx,%edx - 5bc: 69 48 f4 77 ca eb 85 imul $0x85ebca77,-0xc(%rax),%ecx - 5c3: c1 c2 0d rol $0xd,%edx - 5c6: 69 d2 b1 79 37 9e imul $0x9e3779b1,%edx,%edx - 5cc: 01 cf add %ecx,%edi - 5ce: 69 48 f8 77 ca eb 85 imul $0x85ebca77,-0x8(%rax),%ecx - 5d5: c1 c7 0d rol $0xd,%edi - 5d8: 69 ff b1 79 37 9e imul $0x9e3779b1,%edi,%edi - 5de: 01 ce add %ecx,%esi - 5e0: 69 48 fc 77 ca eb 85 imul $0x85ebca77,-0x4(%rax),%ecx - 5e7: c1 c6 0d rol $0xd,%esi - 5ea: 69 f6 b1 79 37 9e imul $0x9e3779b1,%esi,%esi - 5f0: 44 01 d9 add %r11d,%ecx - 5f3: c1 c1 0d rol $0xd,%ecx - 5f6: 44 69 d9 b1 79 37 9e imul $0x9e3779b1,%ecx,%r11d - 5fd: 49 39 c2 cmp %rax,%r10 - 600: 73 ae jae 5b0 <cal_tag_hash_xxhash+0x260> - 602: 89 74 24 18 mov %esi,0x18(%rsp) - 606: 66 0f 6e 4c 24 18 movd 0x18(%rsp),%xmm1 - 60c: 4d 29 f2 sub %r14,%r10 - 60f: 44 89 5c 24 18 mov %r11d,0x18(%rsp) - 614: 66 0f 6e 64 24 18 movd 0x18(%rsp),%xmm4 - 61a: 49 83 e2 f0 and $0xfffffffffffffff0,%r10 - 61e: 89 54 24 18 mov %edx,0x18(%rsp) - 622: 66 0f 6e 44 24 18 movd 0x18(%rsp),%xmm0 - 628: 4f 8d 74 16 10 lea 0x10(%r14,%r10,1),%r14 - 62d: 89 7c 24 18 mov %edi,0x18(%rsp) - 631: 66 0f 6e 6c 24 18 movd 0x18(%rsp),%xmm5 - 637: 66 0f 62 cc punpckldq %xmm4,%xmm1 - 63b: 66 0f 62 c5 punpckldq %xmm5,%xmm0 - 63f: 66 0f 6c c1 punpcklqdq %xmm1,%xmm0 - 643: 0f 11 44 24 28 movups %xmm0,0x28(%rsp) - 648: 45 31 ed xor %r13d,%r13d - 64b: 4d 39 f7 cmp %r14,%r15 - 64e: 0f 87 4c 06 00 00 ja ca0 <cal_tag_hash_xxhash+0x950> - 654: 0f 1f 40 00 nopl 0x0(%rax) - 658: 48 83 c3 18 add $0x18,%rbx - 65c: 48 3b 5c 24 10 cmp 0x10(%rsp),%rbx - 661: 0f 85 89 fd ff ff jne 3f0 <cal_tag_hash_xxhash+0xa0> - 667: 8b 74 24 30 mov 0x30(%rsp),%esi - 66b: 8d 86 b1 67 56 16 lea 0x165667b1(%rsi),%eax - 671: 85 ed test %ebp,%ebp - 673: 0f 85 8f 04 00 00 jne b08 <cal_tag_hash_xxhash+0x7b8> - 679: 46 8d 0c 20 lea (%rax,%r12,1),%r9d - 67d: 48 8b 5c 24 08 mov 0x8(%rsp),%rbx - 682: 44 89 e8 mov %r13d,%eax - 685: 83 e0 0f and $0xf,%eax - 688: 41 83 e5 0c and $0xc,%r13d - 68c: 48 8d 73 18 lea 0x18(%rbx),%rsi - 690: 74 6e je 700 <cal_tag_hash_xxhash+0x3b0> - 692: 69 54 24 38 3d ae b2 imul $0xc2b2ae3d,0x38(%rsp),%edx - 699: c2 - 69a: 48 8d 48 fc lea -0x4(%rax),%rcx - 69e: 41 01 d1 add %edx,%r9d - 6a1: 41 c1 c9 0f ror $0xf,%r9d - 6a5: 45 69 c9 2f eb d4 27 imul $0x27d4eb2f,%r9d,%r9d - 6ac: 48 83 f9 03 cmp $0x3,%rcx - 6b0: 76 36 jbe 6e8 <cal_tag_hash_xxhash+0x398> - 6b2: 69 54 24 3c 3d ae b2 imul $0xc2b2ae3d,0x3c(%rsp),%edx - 6b9: c2 - 6ba: 48 83 e8 08 sub $0x8,%rax - 6be: 41 01 d1 add %edx,%r9d - 6c1: 41 c1 c9 0f ror $0xf,%r9d - 6c5: 45 69 c9 2f eb d4 27 imul $0x27d4eb2f,%r9d,%r9d - 6cc: 48 83 f8 03 cmp $0x3,%rax - 6d0: 76 16 jbe 6e8 <cal_tag_hash_xxhash+0x398> - 6d2: 69 44 24 40 3d ae b2 imul $0xc2b2ae3d,0x40(%rsp),%eax - 6d9: c2 - 6da: 41 01 c1 add %eax,%r9d - 6dd: 41 c1 c9 0f ror $0xf,%r9d - 6e1: 45 69 c9 2f eb d4 27 imul $0x27d4eb2f,%r9d,%r9d - 6e8: 48 89 c8 mov %rcx,%rax - 6eb: 48 8b 5c 24 08 mov 0x8(%rsp),%rbx - 6f0: 48 c1 e8 02 shr $0x2,%rax - 6f4: 48 8d 74 83 1c lea 0x1c(%rbx,%rax,4),%rsi - 6f9: 48 f7 d8 neg %rax - 6fc: 48 8d 04 81 lea (%rcx,%rax,4),%rax - 700: 48 85 c0 test %rax,%rax - 703: 74 53 je 758 <cal_tag_hash_xxhash+0x408> - 705: 0f b6 16 movzbl (%rsi),%edx - 708: 69 d2 b1 67 56 16 imul $0x165667b1,%edx,%edx - 70e: 41 01 d1 add %edx,%r9d - 711: 41 c1 c1 0b rol $0xb,%r9d - 715: 45 69 c9 b1 79 37 9e imul $0x9e3779b1,%r9d,%r9d - 71c: 48 83 e8 01 sub $0x1,%rax - 720: 74 36 je 758 <cal_tag_hash_xxhash+0x408> - 722: 0f b6 56 01 movzbl 0x1(%rsi),%edx - 726: 69 d2 b1 67 56 16 imul $0x165667b1,%edx,%edx - 72c: 41 01 d1 add %edx,%r9d - 72f: 41 c1 c1 0b rol $0xb,%r9d - 733: 45 69 c9 b1 79 37 9e imul $0x9e3779b1,%r9d,%r9d - 73a: 48 83 f8 01 cmp $0x1,%rax - 73e: 74 18 je 758 <cal_tag_hash_xxhash+0x408> - 740: 0f b6 46 02 movzbl 0x2(%rsi),%eax - 744: 69 c0 b1 67 56 16 imul $0x165667b1,%eax,%eax - 74a: 41 01 c1 add %eax,%r9d - 74d: 41 c1 c1 0b rol $0xb,%r9d - 751: 45 69 c9 b1 79 37 9e imul $0x9e3779b1,%r9d,%r9d - 758: 44 89 c8 mov %r9d,%eax - 75b: 48 81 c4 88 00 00 00 add $0x88,%rsp - 762: c1 e8 0f shr $0xf,%eax - 765: 5b pop %rbx - 766: 5d pop %rbp - 767: 41 31 c1 xor %eax,%r9d - 76a: 41 5c pop %r12 - 76c: 41 5d pop %r13 - 76e: 41 69 c1 77 ca eb 85 imul $0x85ebca77,%r9d,%eax - 775: 41 5e pop %r14 - 777: 41 5f pop %r15 - 779: 41 89 c1 mov %eax,%r9d - 77c: 41 c1 e9 0d shr $0xd,%r9d - 780: 44 31 c8 xor %r9d,%eax - 783: 69 c0 3d ae b2 c2 imul $0xc2b2ae3d,%eax,%eax - 789: 89 c2 mov %eax,%edx - 78b: c1 ea 10 shr $0x10,%edx - 78e: 31 d0 xor %edx,%eax - 790: c3 retq - 791: 0f 1f 80 00 00 00 00 nopl 0x0(%rax) - 798: 48 8b 74 24 08 mov 0x8(%rsp),%rsi - 79d: 4c 89 f2 mov %r14,%rdx - 7a0: 48 8d 7c 06 18 lea 0x18(%rsi,%rax,1),%rdi - 7a5: 4c 89 fe mov %r15,%rsi - 7a8: e8 00 00 00 00 callq 7ad <cal_tag_hash_xxhash+0x45d> - 7ad: 44 8b 6c 24 48 mov 0x48(%rsp),%r13d - 7b2: 8b 03 mov (%rbx),%eax - 7b4: 45 01 f5 add %r14d,%r13d - 7b7: 44 89 6c 24 48 mov %r13d,0x48(%rsp) - 7bc: 83 f8 01 cmp $0x1,%eax - 7bf: 0f 85 5a fd ff ff jne 51f <cal_tag_hash_xxhash+0x1cf> - 7c5: 8b 44 24 20 mov 0x20(%rsp),%eax - 7c9: 44 8d 60 08 lea 0x8(%rax),%r12d - 7cd: 31 c0 xor %eax,%eax - 7cf: 41 83 fc 0f cmp $0xf,%r12d - 7d3: 44 89 64 24 20 mov %r12d,0x20(%rsp) - 7d8: 0f 97 c0 seta %al - 7db: 0b 44 24 24 or 0x24(%rsp),%eax - 7df: 89 c5 mov %eax,%ebp - 7e1: 89 44 24 24 mov %eax,0x24(%rsp) - 7e5: 44 89 e8 mov %r13d,%eax - 7e8: 48 8d 50 08 lea 0x8(%rax),%rdx - 7ec: 48 83 fa 0f cmp $0xf,%rdx - 7f0: 0f 86 7a 05 00 00 jbe d70 <cal_tag_hash_xxhash+0xa20> - 7f6: 4c 8d 73 08 lea 0x8(%rbx),%r14 - 7fa: 45 85 ed test %r13d,%r13d - 7fd: 0f 85 f5 03 00 00 jne bf8 <cal_tag_hash_xxhash+0x8a8> - 803: 49 39 de cmp %rbx,%r14 - 806: 0f 87 af 00 00 00 ja 8bb <cal_tag_hash_xxhash+0x56b> - 80c: 8b 74 24 28 mov 0x28(%rsp),%esi - 810: 8b 7c 24 2c mov 0x2c(%rsp),%edi - 814: 4c 89 f0 mov %r14,%rax - 817: 8b 4c 24 30 mov 0x30(%rsp),%ecx - 81b: 44 8b 54 24 34 mov 0x34(%rsp),%r10d - 820: 69 10 77 ca eb 85 imul $0x85ebca77,(%rax),%edx - 826: 48 83 c0 10 add $0x10,%rax - 82a: 01 d6 add %edx,%esi - 82c: 69 50 f4 77 ca eb 85 imul $0x85ebca77,-0xc(%rax),%edx - 833: c1 c6 0d rol $0xd,%esi - 836: 69 f6 b1 79 37 9e imul $0x9e3779b1,%esi,%esi - 83c: 01 d7 add %edx,%edi - 83e: 69 50 f8 77 ca eb 85 imul $0x85ebca77,-0x8(%rax),%edx - 845: c1 c7 0d rol $0xd,%edi - 848: 69 ff b1 79 37 9e imul $0x9e3779b1,%edi,%edi - 84e: 01 d1 add %edx,%ecx - 850: 69 50 fc 77 ca eb 85 imul $0x85ebca77,-0x4(%rax),%edx - 857: c1 c1 0d rol $0xd,%ecx - 85a: 69 c9 b1 79 37 9e imul $0x9e3779b1,%ecx,%ecx - 860: 44 01 d2 add %r10d,%edx - 863: c1 c2 0d rol $0xd,%edx - 866: 44 69 d2 b1 79 37 9e imul $0x9e3779b1,%edx,%r10d - 86d: 48 39 d8 cmp %rbx,%rax - 870: 76 ae jbe 820 <cal_tag_hash_xxhash+0x4d0> - 872: 89 4c 24 18 mov %ecx,0x18(%rsp) - 876: 66 0f 6e 4c 24 18 movd 0x18(%rsp),%xmm1 - 87c: 48 89 d8 mov %rbx,%rax - 87f: 44 89 54 24 18 mov %r10d,0x18(%rsp) - 884: 66 0f 6e 74 24 18 movd 0x18(%rsp),%xmm6 - 88a: 4c 29 f0 sub %r14,%rax - 88d: 89 74 24 18 mov %esi,0x18(%rsp) - 891: 66 0f 6e 44 24 18 movd 0x18(%rsp),%xmm0 - 897: 48 83 e0 f0 and $0xfffffffffffffff0,%rax - 89b: 89 7c 24 18 mov %edi,0x18(%rsp) - 89f: 66 0f 6e 7c 24 18 movd 0x18(%rsp),%xmm7 - 8a5: 66 0f 62 ce punpckldq %xmm6,%xmm1 - 8a9: 4d 8d 74 06 10 lea 0x10(%r14,%rax,1),%r14 - 8ae: 66 0f 62 c7 punpckldq %xmm7,%xmm0 - 8b2: 66 0f 6c c1 punpcklqdq %xmm1,%xmm0 - 8b6: 0f 11 44 24 28 movups %xmm0,0x28(%rsp) - 8bb: 48 8d 53 10 lea 0x10(%rbx),%rdx - 8bf: 45 31 ed xor %r13d,%r13d - 8c2: 49 39 d6 cmp %rdx,%r14 - 8c5: 0f 83 8d fd ff ff jae 658 <cal_tag_hash_xxhash+0x308> - 8cb: 48 8b 44 24 08 mov 0x8(%rsp),%rax - 8d0: 4c 29 f2 sub %r14,%rdx - 8d3: 4c 89 f6 mov %r14,%rsi - 8d6: 49 89 d7 mov %rdx,%r15 - 8d9: 48 8d 78 18 lea 0x18(%rax),%rdi - 8dd: 45 89 fd mov %r15d,%r13d - 8e0: e8 00 00 00 00 callq 8e5 <cal_tag_hash_xxhash+0x595> - 8e5: 44 89 7c 24 48 mov %r15d,0x48(%rsp) - 8ea: e9 69 fd ff ff jmpq 658 <cal_tag_hash_xxhash+0x308> - 8ef: 90 nop - 8f0: 48 8b 43 08 mov 0x8(%rbx),%rax - 8f4: 48 89 44 14 38 mov %rax,0x38(%rsp,%rdx,1) - 8f9: 8b 44 24 48 mov 0x48(%rsp),%eax - 8fd: 44 8d 68 08 lea 0x8(%rax),%r13d - 901: 44 89 6c 24 48 mov %r13d,0x48(%rsp) - 906: 66 2e 0f 1f 84 00 00 nopw %cs:0x0(%rax,%rax,1) - 90d: 00 00 00 - 910: 8b 6c 24 24 mov 0x24(%rsp),%ebp - 914: 44 8b 64 24 20 mov 0x20(%rsp),%r12d - 919: e9 3a fd ff ff jmpq 658 <cal_tag_hash_xxhash+0x308> - 91e: 66 90 xchg %ax,%ax - 920: 8b 44 24 20 mov 0x20(%rsp),%eax - 924: 8b 54 24 48 mov 0x48(%rsp),%edx - 928: 4c 8d 73 08 lea 0x8(%rbx),%r14 - 92c: 4c 8d 7b 10 lea 0x10(%rbx),%r15 - 930: 83 c0 08 add $0x8,%eax - 933: 48 8d 4a 08 lea 0x8(%rdx),%rcx - 937: 83 f8 0f cmp $0xf,%eax - 93a: 89 44 24 20 mov %eax,0x20(%rsp) - 93e: 0f 97 c0 seta %al - 941: 0f b6 c0 movzbl %al,%eax - 944: 09 44 24 24 or %eax,0x24(%rsp) - 948: 48 89 d0 mov %rdx,%rax - 94b: 48 83 f9 0f cmp $0xf,%rcx - 94f: 76 9f jbe 8f0 <cal_tag_hash_xxhash+0x5a0> - 951: 85 d2 test %edx,%edx - 953: 0f 85 07 02 00 00 jne b60 <cal_tag_hash_xxhash+0x810> - 959: 49 39 de cmp %rbx,%r14 - 95c: 0f 87 bc 00 00 00 ja a1e <cal_tag_hash_xxhash+0x6ce> - 962: 8b 54 24 28 mov 0x28(%rsp),%edx - 966: 8b 7c 24 2c mov 0x2c(%rsp),%edi - 96a: 4c 89 f0 mov %r14,%rax - 96d: 44 8b 44 24 30 mov 0x30(%rsp),%r8d - 972: 44 8b 4c 24 34 mov 0x34(%rsp),%r9d - 977: 66 0f 1f 84 00 00 00 nopw 0x0(%rax,%rax,1) - 97e: 00 00 - 980: 69 08 77 ca eb 85 imul $0x85ebca77,(%rax),%ecx - 986: 48 83 c0 10 add $0x10,%rax - 98a: 69 70 f8 77 ca eb 85 imul $0x85ebca77,-0x8(%rax),%esi - 991: 01 ca add %ecx,%edx - 993: 69 48 f4 77 ca eb 85 imul $0x85ebca77,-0xc(%rax),%ecx - 99a: 44 01 c6 add %r8d,%esi - 99d: c1 c2 0d rol $0xd,%edx - 9a0: c1 c6 0d rol $0xd,%esi - 9a3: 69 d2 b1 79 37 9e imul $0x9e3779b1,%edx,%edx - 9a9: 01 cf add %ecx,%edi - 9ab: 69 48 fc 77 ca eb 85 imul $0x85ebca77,-0x4(%rax),%ecx - 9b2: c1 c7 0d rol $0xd,%edi - 9b5: 44 69 c6 b1 79 37 9e imul $0x9e3779b1,%esi,%r8d - 9bc: 69 ff b1 79 37 9e imul $0x9e3779b1,%edi,%edi - 9c2: 44 01 c9 add %r9d,%ecx - 9c5: c1 c1 0d rol $0xd,%ecx - 9c8: 44 69 c9 b1 79 37 9e imul $0x9e3779b1,%ecx,%r9d - 9cf: 48 39 d8 cmp %rbx,%rax - 9d2: 76 ac jbe 980 <cal_tag_hash_xxhash+0x630> - 9d4: 44 89 44 24 18 mov %r8d,0x18(%rsp) - 9d9: 66 0f 6e 4c 24 18 movd 0x18(%rsp),%xmm1 - 9df: 48 89 d8 mov %rbx,%rax - 9e2: 44 89 4c 24 18 mov %r9d,0x18(%rsp) - 9e7: 66 0f 6e 74 24 18 movd 0x18(%rsp),%xmm6 - 9ed: 4c 29 f0 sub %r14,%rax - 9f0: 89 54 24 18 mov %edx,0x18(%rsp) - 9f4: 66 0f 6e 44 24 18 movd 0x18(%rsp),%xmm0 - 9fa: 48 83 e0 f0 and $0xfffffffffffffff0,%rax - 9fe: 89 7c 24 18 mov %edi,0x18(%rsp) - a02: 66 0f 6e 7c 24 18 movd 0x18(%rsp),%xmm7 - a08: 66 0f 62 ce punpckldq %xmm6,%xmm1 - a0c: 4d 8d 74 06 10 lea 0x10(%r14,%rax,1),%r14 - a11: 66 0f 62 c7 punpckldq %xmm7,%xmm0 - a15: 66 0f 6c c1 punpcklqdq %xmm1,%xmm0 - a19: 0f 11 44 24 28 movups %xmm0,0x28(%rsp) - a1e: 4d 39 fe cmp %r15,%r14 - a21: 0f 82 09 01 00 00 jb b30 <cal_tag_hash_xxhash+0x7e0> - a27: 44 8b 6c 24 48 mov 0x48(%rsp),%r13d - a2c: 8b 6c 24 24 mov 0x24(%rsp),%ebp - a30: 44 8b 64 24 20 mov 0x20(%rsp),%r12d - a35: e9 1e fc ff ff jmpq 658 <cal_tag_hash_xxhash+0x308> - a3a: 66 0f 1f 44 00 00 nopw 0x0(%rax,%rax,1) - a40: 48 8b 74 24 08 mov 0x8(%rsp),%rsi - a45: bd 10 00 00 00 mov $0x10,%ebp - a4a: 89 ea mov %ebp,%edx - a4c: 48 8d 7c 06 18 lea 0x18(%rsi,%rax,1),%rdi - a51: 44 29 ea sub %r13d,%edx - a54: 4c 89 fe mov %r15,%rsi - a57: e8 00 00 00 00 callq a5c <cal_tag_hash_xxhash+0x70c> - a5c: 69 44 24 38 77 ca eb imul $0x85ebca77,0x38(%rsp),%eax - a63: 85 - a64: 41 89 e8 mov %ebp,%r8d - a67: 03 44 24 28 add 0x28(%rsp),%eax - a6b: c1 c0 0d rol $0xd,%eax - a6e: 44 2b 44 24 48 sub 0x48(%rsp),%r8d - a73: c7 44 24 48 00 00 00 movl $0x0,0x48(%rsp) - a7a: 00 - a7b: 69 c0 b1 79 37 9e imul $0x9e3779b1,%eax,%eax - a81: 4b 8d 34 07 lea (%r15,%r8,1),%rsi - a85: 89 44 24 28 mov %eax,0x28(%rsp) - a89: 69 44 24 3c 77 ca eb imul $0x85ebca77,0x3c(%rsp),%eax - a90: 85 - a91: 03 44 24 2c add 0x2c(%rsp),%eax - a95: c1 c0 0d rol $0xd,%eax - a98: 69 c0 b1 79 37 9e imul $0x9e3779b1,%eax,%eax - a9e: 89 44 24 2c mov %eax,0x2c(%rsp) - aa2: 69 44 24 40 77 ca eb imul $0x85ebca77,0x40(%rsp),%eax - aa9: 85 - aaa: 03 44 24 30 add 0x30(%rsp),%eax - aae: c1 c0 0d rol $0xd,%eax - ab1: 69 c0 b1 79 37 9e imul $0x9e3779b1,%eax,%eax - ab7: 89 44 24 30 mov %eax,0x30(%rsp) - abb: 69 44 24 44 77 ca eb imul $0x85ebca77,0x44(%rsp),%eax - ac2: 85 - ac3: 03 44 24 34 add 0x34(%rsp),%eax - ac7: c1 c0 0d rol $0xd,%eax - aca: 69 c0 b1 79 37 9e imul $0x9e3779b1,%eax,%eax - ad0: 89 44 24 34 mov %eax,0x34(%rsp) - ad4: e9 65 f9 ff ff jmpq 43e <cal_tag_hash_xxhash+0xee> - ad9: 0f 1f 80 00 00 00 00 nopl 0x0(%rax) - ae0: 48 29 f0 sub %rsi,%rax - ae3: 49 89 c6 mov %rax,%r14 - ae6: 48 8b 44 24 08 mov 0x8(%rsp),%rax - aeb: 4c 89 f2 mov %r14,%rdx - aee: 45 89 f5 mov %r14d,%r13d - af1: 48 8d 78 18 lea 0x18(%rax),%rdi - af5: e8 00 00 00 00 callq afa <cal_tag_hash_xxhash+0x7aa> - afa: 44 89 74 24 48 mov %r14d,0x48(%rsp) - aff: e9 10 fa ff ff jmpq 514 <cal_tag_hash_xxhash+0x1c4> - b04: 0f 1f 40 00 nopl 0x0(%rax) - b08: 8b 54 24 28 mov 0x28(%rsp),%edx - b0c: 8b 44 24 2c mov 0x2c(%rsp),%eax - b10: c1 c6 0c rol $0xc,%esi - b13: c1 c0 07 rol $0x7,%eax - b16: d1 c2 rol %edx - b18: 01 c2 add %eax,%edx - b1a: 8d 04 32 lea (%rdx,%rsi,1),%eax - b1d: 8b 54 24 34 mov 0x34(%rsp),%edx - b21: c1 ca 0e ror $0xe,%edx - b24: 01 d0 add %edx,%eax - b26: e9 4e fb ff ff jmpq 679 <cal_tag_hash_xxhash+0x329> - b2b: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) - b30: 48 8b 44 24 08 mov 0x8(%rsp),%rax - b35: 4d 29 f7 sub %r14,%r15 - b38: 4c 89 f6 mov %r14,%rsi - b3b: 4c 89 fa mov %r15,%rdx - b3e: 45 89 fd mov %r15d,%r13d - b41: 48 8d 78 18 lea 0x18(%rax),%rdi - b45: e8 00 00 00 00 callq b4a <cal_tag_hash_xxhash+0x7fa> - b4a: 44 89 7c 24 48 mov %r15d,0x48(%rsp) - b4f: 8b 6c 24 24 mov 0x24(%rsp),%ebp - b53: 44 8b 64 24 20 mov 0x20(%rsp),%r12d - b58: e9 fb fa ff ff jmpq 658 <cal_tag_hash_xxhash+0x308> - b5d: 0f 1f 00 nopl (%rax) - b60: 48 8b 74 24 08 mov 0x8(%rsp),%rsi - b65: bd 10 00 00 00 mov $0x10,%ebp - b6a: 48 8d 7c 16 18 lea 0x18(%rsi,%rdx,1),%rdi - b6f: 89 ea mov %ebp,%edx - b71: 4c 89 f6 mov %r14,%rsi - b74: 29 c2 sub %eax,%edx - b76: e8 00 00 00 00 callq b7b <cal_tag_hash_xxhash+0x82b> - b7b: 69 44 24 38 77 ca eb imul $0x85ebca77,0x38(%rsp),%eax - b82: 85 - b83: 89 e9 mov %ebp,%ecx - b85: 03 44 24 28 add 0x28(%rsp),%eax - b89: c1 c0 0d rol $0xd,%eax - b8c: 2b 4c 24 48 sub 0x48(%rsp),%ecx - b90: c7 44 24 48 00 00 00 movl $0x0,0x48(%rsp) - b97: 00 - b98: 69 c0 b1 79 37 9e imul $0x9e3779b1,%eax,%eax - b9e: 49 01 ce add %rcx,%r14 - ba1: 89 44 24 28 mov %eax,0x28(%rsp) - ba5: 69 44 24 3c 77 ca eb imul $0x85ebca77,0x3c(%rsp),%eax - bac: 85 - bad: 03 44 24 2c add 0x2c(%rsp),%eax - bb1: c1 c0 0d rol $0xd,%eax - bb4: 69 c0 b1 79 37 9e imul $0x9e3779b1,%eax,%eax - bba: 89 44 24 2c mov %eax,0x2c(%rsp) - bbe: 69 44 24 40 77 ca eb imul $0x85ebca77,0x40(%rsp),%eax - bc5: 85 - bc6: 03 44 24 30 add 0x30(%rsp),%eax - bca: c1 c0 0d rol $0xd,%eax - bcd: 69 c0 b1 79 37 9e imul $0x9e3779b1,%eax,%eax - bd3: 89 44 24 30 mov %eax,0x30(%rsp) - bd7: 69 44 24 44 77 ca eb imul $0x85ebca77,0x44(%rsp),%eax - bde: 85 - bdf: 03 44 24 34 add 0x34(%rsp),%eax - be3: c1 c0 0d rol $0xd,%eax - be6: 69 c0 b1 79 37 9e imul $0x9e3779b1,%eax,%eax - bec: 89 44 24 34 mov %eax,0x34(%rsp) - bf0: e9 64 fd ff ff jmpq 959 <cal_tag_hash_xxhash+0x609> - bf5: 0f 1f 00 nopl (%rax) - bf8: 48 8b 74 24 08 mov 0x8(%rsp),%rsi - bfd: 41 bf 10 00 00 00 mov $0x10,%r15d - c03: 44 89 fa mov %r15d,%edx - c06: 48 8d 7c 06 18 lea 0x18(%rsi,%rax,1),%rdi - c0b: 44 29 ea sub %r13d,%edx - c0e: 4c 89 f6 mov %r14,%rsi - c11: e8 00 00 00 00 callq c16 <cal_tag_hash_xxhash+0x8c6> - c16: 44 2b 7c 24 48 sub 0x48(%rsp),%r15d - c1b: 8b 6c 24 24 mov 0x24(%rsp),%ebp - c1f: c7 44 24 48 00 00 00 movl $0x0,0x48(%rsp) - c26: 00 - c27: 69 44 24 38 77 ca eb imul $0x85ebca77,0x38(%rsp),%eax - c2e: 85 - c2f: 03 44 24 28 add 0x28(%rsp),%eax - c33: 4d 01 fe add %r15,%r14 - c36: c1 c0 0d rol $0xd,%eax - c39: 44 8b 64 24 20 mov 0x20(%rsp),%r12d - c3e: 69 c0 b1 79 37 9e imul $0x9e3779b1,%eax,%eax - c44: 89 44 24 28 mov %eax,0x28(%rsp) - c48: 69 44 24 3c 77 ca eb imul $0x85ebca77,0x3c(%rsp),%eax - c4f: 85 - c50: 03 44 24 2c add 0x2c(%rsp),%eax - c54: c1 c0 0d rol $0xd,%eax - c57: 69 c0 b1 79 37 9e imul $0x9e3779b1,%eax,%eax - c5d: 89 44 24 2c mov %eax,0x2c(%rsp) - c61: 69 44 24 40 77 ca eb imul $0x85ebca77,0x40(%rsp),%eax - c68: 85 - c69: 03 44 24 30 add 0x30(%rsp),%eax - c6d: c1 c0 0d rol $0xd,%eax - c70: 69 c0 b1 79 37 9e imul $0x9e3779b1,%eax,%eax - c76: 89 44 24 30 mov %eax,0x30(%rsp) - c7a: 69 44 24 44 77 ca eb imul $0x85ebca77,0x44(%rsp),%eax - c81: 85 - c82: 03 44 24 34 add 0x34(%rsp),%eax - c86: c1 c0 0d rol $0xd,%eax - c89: 69 c0 b1 79 37 9e imul $0x9e3779b1,%eax,%eax - c8f: 89 44 24 34 mov %eax,0x34(%rsp) - c93: e9 6b fb ff ff jmpq 803 <cal_tag_hash_xxhash+0x4b3> - c98: 0f 1f 84 00 00 00 00 nopl 0x0(%rax,%rax,1) - c9f: 00 - ca0: 48 8b 44 24 08 mov 0x8(%rsp),%rax - ca5: 4d 29 f7 sub %r14,%r15 - ca8: 4c 89 f6 mov %r14,%rsi - cab: 4c 89 fa mov %r15,%rdx - cae: 45 89 fd mov %r15d,%r13d - cb1: 48 8d 78 18 lea 0x18(%rax),%rdi - cb5: e8 00 00 00 00 callq cba <cal_tag_hash_xxhash+0x96a> - cba: 44 89 7c 24 48 mov %r15d,0x48(%rsp) - cbf: e9 94 f9 ff ff jmpq 658 <cal_tag_hash_xxhash+0x308> - cc4: 0f 1f 40 00 nopl 0x0(%rax) - cc8: 48 8b 74 24 08 mov 0x8(%rsp),%rsi - ccd: bd 10 00 00 00 mov $0x10,%ebp - cd2: 89 ea mov %ebp,%edx - cd4: 48 8d 7c 06 18 lea 0x18(%rsi,%rax,1),%rdi - cd9: 44 29 ea sub %r13d,%edx - cdc: 4c 89 f6 mov %r14,%rsi - cdf: e8 00 00 00 00 callq ce4 <cal_tag_hash_xxhash+0x994> - ce4: 69 44 24 38 77 ca eb imul $0x85ebca77,0x38(%rsp),%eax - ceb: 85 - cec: 03 44 24 28 add 0x28(%rsp),%eax - cf0: 41 89 e8 mov %ebp,%r8d - cf3: c1 c0 0d rol $0xd,%eax - cf6: 44 2b 44 24 48 sub 0x48(%rsp),%r8d - cfb: 8b 6c 24 24 mov 0x24(%rsp),%ebp - cff: c7 44 24 48 00 00 00 movl $0x0,0x48(%rsp) - d06: 00 - d07: 69 c0 b1 79 37 9e imul $0x9e3779b1,%eax,%eax - d0d: 4d 01 c6 add %r8,%r14 - d10: 44 8b 64 24 20 mov 0x20(%rsp),%r12d - d15: 89 44 24 28 mov %eax,0x28(%rsp) - d19: 69 44 24 3c 77 ca eb imul $0x85ebca77,0x3c(%rsp),%eax - d20: 85 - d21: 03 44 24 2c add 0x2c(%rsp),%eax - d25: c1 c0 0d rol $0xd,%eax - d28: 69 c0 b1 79 37 9e imul $0x9e3779b1,%eax,%eax - d2e: 89 44 24 2c mov %eax,0x2c(%rsp) - d32: 69 44 24 40 77 ca eb imul $0x85ebca77,0x40(%rsp),%eax - d39: 85 - d3a: 03 44 24 30 add 0x30(%rsp),%eax - d3e: c1 c0 0d rol $0xd,%eax - d41: 69 c0 b1 79 37 9e imul $0x9e3779b1,%eax,%eax - d47: 89 44 24 30 mov %eax,0x30(%rsp) - d4b: 69 44 24 44 77 ca eb imul $0x85ebca77,0x44(%rsp),%eax - d52: 85 - d53: 03 44 24 34 add 0x34(%rsp),%eax - d57: c1 c0 0d rol $0xd,%eax - d5a: 69 c0 b1 79 37 9e imul $0x9e3779b1,%eax,%eax - d60: 89 44 24 34 mov %eax,0x34(%rsp) - d64: e9 20 f8 ff ff jmpq 589 <cal_tag_hash_xxhash+0x239> - d69: 0f 1f 80 00 00 00 00 nopl 0x0(%rax) - d70: 48 8b 53 08 mov 0x8(%rbx),%rdx - d74: 48 89 54 04 38 mov %rdx,0x38(%rsp,%rax,1) - d79: e9 7b fb ff ff jmpq 8f9 <cal_tag_hash_xxhash+0x5a9> - d7e: 66 90 xchg %ax,%ax - d80: 48 8b 74 24 08 mov 0x8(%rsp),%rsi - d85: 4c 89 d2 mov %r10,%rdx - d88: 4c 89 54 24 18 mov %r10,0x18(%rsp) - d8d: 48 8d 7c 06 18 lea 0x18(%rsi,%rax,1),%rdi - d92: 4c 89 f6 mov %r14,%rsi - d95: e8 00 00 00 00 callq d9a <cal_tag_hash_xxhash+0xa4a> - d9a: 44 8b 6c 24 48 mov 0x48(%rsp),%r13d - d9f: 4c 8b 54 24 18 mov 0x18(%rsp),%r10 - da4: 8b 6c 24 24 mov 0x24(%rsp),%ebp - da8: 44 8b 64 24 20 mov 0x20(%rsp),%r12d - dad: 45 01 d5 add %r10d,%r13d - db0: 44 89 6c 24 48 mov %r13d,0x48(%rsp) - db5: e9 9e f8 ff ff jmpq 658 <cal_tag_hash_xxhash+0x308> - dba: 66 0f 1f 44 00 00 nopw 0x0(%rax,%rax,1) - dc0: 8b 74 24 30 mov 0x30(%rsp),%esi - dc4: 8b 6c 24 24 mov 0x24(%rsp),%ebp - dc8: 48 8d 44 24 20 lea 0x20(%rsp),%rax - dcd: 48 89 44 24 08 mov %rax,0x8(%rsp) - dd2: 44 8b 64 24 20 mov 0x20(%rsp),%r12d - dd7: 8d 86 b1 67 56 16 lea 0x165667b1(%rsi),%eax - ddd: 85 ed test %ebp,%ebp - ddf: 0f 84 94 f8 ff ff je 679 <cal_tag_hash_xxhash+0x329> - de5: e9 1e fd ff ff jmpq b08 <cal_tag_hash_xxhash+0x7b8> - dea: 66 0f 1f 44 00 00 nopw 0x0(%rax,%rax,1) - -0000000000000df0 <my_tag_hash_func>: - df0: 31 d2 xor %edx,%edx - df2: e9 00 00 00 00 jmpq df7 <my_tag_hash_func+0x7> - df7: 66 0f 1f 84 00 00 00 nopw 0x0(%rax,%rax,1) - dfe: 00 00 - -0000000000000e00 <tag_hash_key_construct_with_fieldstat_tag>: - e00: 41 54 push %r12 - e02: 49 89 f4 mov %rsi,%r12 - e05: 55 push %rbp - e06: 48 89 fd mov %rdi,%rbp - e09: bf 18 00 00 00 mov $0x18,%edi - e0e: 53 push %rbx - e0f: e8 00 00 00 00 callq e14 <tag_hash_key_construct_with_fieldstat_tag+0x14> - e14: 4d 85 e4 test %r12,%r12 - e17: 4c 89 e6 mov %r12,%rsi - e1a: 48 89 c3 mov %rax,%rbx - e1d: b8 00 00 00 00 mov $0x0,%eax - e22: 48 0f 44 e8 cmove %rax,%rbp - e26: 4c 89 63 08 mov %r12,0x8(%rbx) - e2a: c6 43 14 00 movb $0x0,0x14(%rbx) - e2e: 48 89 ef mov %rbp,%rdi - e31: 48 89 2b mov %rbp,(%rbx) - e34: e8 00 00 00 00 callq e39 <tag_hash_key_construct_with_fieldstat_tag+0x39> - e39: 89 43 10 mov %eax,0x10(%rbx) - e3c: 48 89 d8 mov %rbx,%rax - e3f: 5b pop %rbx - e40: 5d pop %rbp - e41: 41 5c pop %r12 - e43: c3 retq - e44: 66 66 2e 0f 1f 84 00 data16 nopw %cs:0x0(%rax,%rax,1) - e4b: 00 00 00 00 - e4f: 90 nop - -0000000000000e50 <tag_hash_key_store>: - e50: 80 7f 14 00 cmpb $0x0,0x14(%rdi) - e54: 75 0d jne e63 <tag_hash_key_store+0x13> - e56: 48 8b 47 08 mov 0x8(%rdi),%rax - e5a: c6 47 14 01 movb $0x1,0x14(%rdi) - e5e: 48 85 c0 test %rax,%rax - e61: 75 05 jne e68 <tag_hash_key_store+0x18> - e63: c3 retq - e64: 0f 1f 40 00 nopl 0x0(%rax) - e68: 41 55 push %r13 - e6a: 41 54 push %r12 - e6c: 45 31 e4 xor %r12d,%r12d - e6f: 55 push %rbp - e70: 48 89 fd mov %rdi,%rbp - e73: 48 8d 3c 40 lea (%rax,%rax,2),%rdi - e77: 53 push %rbx - e78: 48 c1 e7 03 shl $0x3,%rdi - e7c: 31 db xor %ebx,%ebx - e7e: 48 83 ec 08 sub $0x8,%rsp - e82: e8 00 00 00 00 callq e87 <tag_hash_key_store+0x37> - e87: 49 89 c5 mov %rax,%r13 - e8a: 66 0f 1f 44 00 00 nopw 0x0(%rax,%rax,1) - e90: 48 8b 75 00 mov 0x0(%rbp),%rsi - e94: 49 8d 7c 1d 00 lea 0x0(%r13,%rbx,1),%rdi - e99: 49 83 c4 01 add $0x1,%r12 - e9d: 48 01 de add %rbx,%rsi - ea0: 48 83 c3 18 add $0x18,%rbx - ea4: e8 00 00 00 00 callq ea9 <tag_hash_key_store+0x59> - ea9: 4c 39 65 08 cmp %r12,0x8(%rbp) - ead: 77 e1 ja e90 <tag_hash_key_store+0x40> - eaf: 4c 89 6d 00 mov %r13,0x0(%rbp) - eb3: 48 83 c4 08 add $0x8,%rsp - eb7: 5b pop %rbx - eb8: 5d pop %rbp - eb9: 41 5c pop %r12 - ebb: 41 5d pop %r13 - ebd: c3 retq - ebe: 66 90 xchg %ax,%ax - -0000000000000ec0 <tag_hash_key_convert_to_fieldstat_tag>: - ec0: 48 8b 47 08 mov 0x8(%rdi),%rax - ec4: 48 89 02 mov %rax,(%rdx) - ec7: 48 85 c0 test %rax,%rax - eca: 75 0c jne ed8 <tag_hash_key_convert_to_fieldstat_tag+0x18> - ecc: 48 c7 06 00 00 00 00 movq $0x0,(%rsi) - ed3: c3 retq - ed4: 0f 1f 40 00 nopl 0x0(%rax) - ed8: 41 55 push %r13 - eda: 41 54 push %r12 - edc: 4c 8d 24 40 lea (%rax,%rax,2),%r12 - ee0: 55 push %rbp - ee1: 49 c1 e4 03 shl $0x3,%r12 - ee5: 48 89 fd mov %rdi,%rbp - ee8: 53 push %rbx - ee9: 4c 89 e7 mov %r12,%rdi - eec: 48 89 f3 mov %rsi,%rbx - eef: 48 83 ec 08 sub $0x8,%rsp - ef3: e8 00 00 00 00 callq ef8 <tag_hash_key_convert_to_fieldstat_tag+0x38> - ef8: 48 89 03 mov %rax,(%rbx) - efb: 49 89 c5 mov %rax,%r13 - efe: 31 db xor %ebx,%ebx - f00: 48 8b 75 00 mov 0x0(%rbp),%rsi - f04: 49 8d 7c 1d 00 lea 0x0(%r13,%rbx,1),%rdi - f09: 48 01 de add %rbx,%rsi - f0c: 48 83 c3 18 add $0x18,%rbx - f10: e8 00 00 00 00 callq f15 <tag_hash_key_convert_to_fieldstat_tag+0x55> - f15: 49 39 dc cmp %rbx,%r12 - f18: 75 e6 jne f00 <tag_hash_key_convert_to_fieldstat_tag+0x40> - f1a: 48 83 c4 08 add $0x8,%rsp - f1e: 5b pop %rbx - f1f: 5d pop %rbp - f20: 41 5c pop %r12 - f22: 41 5d pop %r13 - f24: c3 retq - f25: 66 66 2e 0f 1f 84 00 data16 nopw %cs:0x0(%rax,%rax,1) - f2c: 00 00 00 00 - -0000000000000f30 <tag_hash_key_copy>: - f30: 41 54 push %r12 - f32: 55 push %rbp - f33: 48 89 fd mov %rdi,%rbp - f36: bf 18 00 00 00 mov $0x18,%edi - f3b: 53 push %rbx - f3c: e8 00 00 00 00 callq f41 <tag_hash_key_copy+0x11> - f41: 8b 55 10 mov 0x10(%rbp),%edx - f44: 49 89 c4 mov %rax,%r12 - f47: 48 8b 45 08 mov 0x8(%rbp),%rax - f4b: 41 89 54 24 10 mov %edx,0x10(%r12) - f50: 49 89 44 24 08 mov %rax,0x8(%r12) - f55: 41 c6 44 24 14 01 movb $0x1,0x14(%r12) - f5b: 48 85 c0 test %rax,%rax - f5e: 75 10 jne f70 <tag_hash_key_copy+0x40> - f60: 4c 89 e0 mov %r12,%rax - f63: 5b pop %rbx - f64: 5d pop %rbp - f65: 49 c7 04 24 00 00 00 movq $0x0,(%r12) - f6c: 00 - f6d: 41 5c pop %r12 - f6f: c3 retq - f70: 48 8d 3c 40 lea (%rax,%rax,2),%rdi - f74: 31 db xor %ebx,%ebx - f76: 48 c1 e7 03 shl $0x3,%rdi - f7a: e8 00 00 00 00 callq f7f <tag_hash_key_copy+0x4f> - f7f: 49 89 04 24 mov %rax,(%r12) - f83: eb 07 jmp f8c <tag_hash_key_copy+0x5c> - f85: 0f 1f 00 nopl (%rax) - f88: 49 8b 04 24 mov (%r12),%rax - f8c: 48 8d 3c 5b lea (%rbx,%rbx,2),%rdi - f90: 48 8b 75 00 mov 0x0(%rbp),%rsi - f94: 48 83 c3 01 add $0x1,%rbx - f98: 48 c1 e7 03 shl $0x3,%rdi - f9c: 48 01 fe add %rdi,%rsi - f9f: 48 01 c7 add %rax,%rdi - fa2: e8 00 00 00 00 callq fa7 <tag_hash_key_copy+0x77> - fa7: 49 39 5c 24 08 cmp %rbx,0x8(%r12) - fac: 77 da ja f88 <tag_hash_key_copy+0x58> - fae: 4c 89 e0 mov %r12,%rax - fb1: 5b pop %rbx - fb2: 5d pop %rbp - fb3: 41 5c pop %r12 - fb5: c3 retq - fb6: 66 2e 0f 1f 84 00 00 nopw %cs:0x0(%rax,%rax,1) - fbd: 00 00 00 - -0000000000000fc0 <tag_hash_key_free>: - fc0: 53 push %rbx - fc1: 80 7f 14 00 cmpb $0x0,0x14(%rdi) - fc5: 48 89 fb mov %rdi,%rbx - fc8: 75 0e jne fd8 <tag_hash_key_free+0x18> - fca: 48 89 df mov %rbx,%rdi - fcd: 5b pop %rbx - fce: e9 00 00 00 00 jmpq fd3 <tag_hash_key_free+0x13> - fd3: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) - fd8: 48 8b 77 08 mov 0x8(%rdi),%rsi - fdc: 48 8b 3f mov (%rdi),%rdi - fdf: e8 00 00 00 00 callq fe4 <tag_hash_key_free+0x24> - fe4: 48 89 df mov %rbx,%rdi - fe7: 5b pop %rbx - fe8: e9 00 00 00 00 jmpq fed <tag_hash_key_free+0x2d> - fed: 0f 1f 00 nopl (%rax) - -0000000000000ff0 <tag_hash_key_cal_hash_val>: - ff0: 89 f2 mov %esi,%edx - ff2: 48 8b 77 08 mov 0x8(%rdi),%rsi - ff6: 48 8b 3f mov (%rdi),%rdi - ff9: e9 00 00 00 00 jmpq ffe <tag_hash_key_cal_hash_val+0xe> - ffe: 66 90 xchg %ax,%ax - -0000000000001000 <build_dynamic_cell_key>: - 1000: 41 57 push %r15 - 1002: 41 56 push %r14 - 1004: 41 55 push %r13 - 1006: 41 54 push %r12 - 1008: 55 push %rbp - 1009: 53 push %rbx - 100a: 48 83 ec 58 sub $0x58,%rsp - 100e: 48 89 54 24 48 mov %rdx,0x48(%rsp) - 1013: 48 89 4c 24 40 mov %rcx,0x40(%rsp) - 1018: 48 85 f6 test %rsi,%rsi - 101b: 0f 84 a8 02 00 00 je 12c9 <build_dynamic_cell_key+0x2c9> - 1021: 48 89 fd mov %rdi,%rbp - 1024: bf 00 04 00 00 mov $0x400,%edi - 1029: 48 89 f3 mov %rsi,%rbx - 102c: e8 00 00 00 00 callq 1031 <build_dynamic_cell_key+0x31> - 1031: bf 00 04 00 00 mov $0x400,%edi - 1036: 49 89 c5 mov %rax,%r13 - 1039: e8 00 00 00 00 callq 103e <build_dynamic_cell_key+0x3e> - 103e: 49 89 c4 mov %rax,%r12 - 1041: 85 db test %ebx,%ebx - 1043: 0f 8e e8 02 00 00 jle 1331 <build_dynamic_cell_key+0x331> - 1049: 8d 43 ff lea -0x1(%rbx),%eax - 104c: 48 89 6c 24 08 mov %rbp,0x8(%rsp) - 1051: 41 be 00 04 00 00 mov $0x400,%r14d - 1057: bb 00 04 00 00 mov $0x400,%ebx - 105c: 48 8d 04 40 lea (%rax,%rax,2),%rax - 1060: c7 44 24 24 00 00 00 movl $0x0,0x24(%rsp) - 1067: 00 - 1068: 48 c7 44 24 10 00 04 movq $0x400,0x10(%rsp) - 106f: 00 00 - 1071: 48 8d 44 c5 18 lea 0x18(%rbp,%rax,8),%rax - 1076: 48 89 44 24 18 mov %rax,0x18(%rsp) - 107b: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) - 1080: 48 8b 44 24 08 mov 0x8(%rsp),%rax - 1085: 8b 40 08 mov 0x8(%rax),%eax - 1088: 83 f8 01 cmp $0x1,%eax - 108b: 0f 84 ef 01 00 00 je 1280 <build_dynamic_cell_key+0x280> - 1091: 85 c0 test %eax,%eax - 1093: 0f 84 77 01 00 00 je 1210 <build_dynamic_cell_key+0x210> - 1099: 83 f8 02 cmp $0x2,%eax - 109c: 0f 84 de 00 00 00 je 1180 <build_dynamic_cell_key+0x180> - 10a2: 48 63 6c 24 24 movslq 0x24(%rsp),%rbp - 10a7: 4c 8b 7c 24 10 mov 0x10(%rsp),%r15 - 10ac: 49 8d 86 00 04 00 00 lea 0x400(%r14),%rax - 10b3: 4c 29 f8 sub %r15,%rax - 10b6: 48 89 c2 mov %rax,%rdx - 10b9: 4c 39 fd cmp %r15,%rbp - 10bc: 72 33 jb 10f1 <build_dynamic_cell_key+0xf1> - 10be: 4c 89 e0 mov %r12,%rax - 10c1: 49 89 dc mov %rbx,%r12 - 10c4: 48 89 d3 mov %rdx,%rbx - 10c7: 66 0f 1f 84 00 00 00 nopw 0x0(%rax,%rax,1) - 10ce: 00 00 - 10d0: 4e 8d 34 3b lea (%rbx,%r15,1),%r14 - 10d4: 48 89 c7 mov %rax,%rdi - 10d7: 49 81 c7 00 04 00 00 add $0x400,%r15 - 10de: 4c 89 f6 mov %r14,%rsi - 10e1: e8 00 00 00 00 callq 10e6 <build_dynamic_cell_key+0xe6> - 10e6: 49 39 ef cmp %rbp,%r15 - 10e9: 76 e5 jbe 10d0 <build_dynamic_cell_key+0xd0> - 10eb: 4c 89 e3 mov %r12,%rbx - 10ee: 49 89 c4 mov %rax,%r12 - 10f1: 4c 89 f7 mov %r14,%rdi - 10f4: 48 89 ea mov %rbp,%rdx - 10f7: 4c 89 ee mov %r13,%rsi - 10fa: 4c 29 ff sub %r15,%rdi - 10fd: 4c 01 e7 add %r12,%rdi - 1100: e8 00 00 00 00 callq 1105 <build_dynamic_cell_key+0x105> - 1105: 4c 89 f8 mov %r15,%rax - 1108: 48 83 44 24 08 18 addq $0x18,0x8(%rsp) - 110e: 48 29 e8 sub %rbp,%rax - 1111: 48 89 44 24 10 mov %rax,0x10(%rsp) - 1116: 48 8b 44 24 08 mov 0x8(%rsp),%rax - 111b: 48 39 44 24 18 cmp %rax,0x18(%rsp) - 1120: 0f 85 5a ff ff ff jne 1080 <build_dynamic_cell_key+0x80> - 1126: 4c 29 fd sub %r15,%rbp - 1129: 4c 01 f5 add %r14,%rbp - 112c: 48 8d 7d 01 lea 0x1(%rbp),%rdi - 1130: e8 00 00 00 00 callq 1135 <build_dynamic_cell_key+0x135> - 1135: 48 89 ea mov %rbp,%rdx - 1138: 4c 89 e6 mov %r12,%rsi - 113b: 48 89 c1 mov %rax,%rcx - 113e: 48 8b 44 24 48 mov 0x48(%rsp),%rax - 1143: 48 89 cf mov %rcx,%rdi - 1146: 48 89 08 mov %rcx,(%rax) - 1149: e8 00 00 00 00 callq 114e <build_dynamic_cell_key+0x14e> - 114e: 4c 89 e7 mov %r12,%rdi - 1151: c6 04 28 00 movb $0x0,(%rax,%rbp,1) - 1155: 48 8b 44 24 40 mov 0x40(%rsp),%rax - 115a: 48 89 28 mov %rbp,(%rax) - 115d: e8 00 00 00 00 callq 1162 <build_dynamic_cell_key+0x162> - 1162: 48 83 c4 58 add $0x58,%rsp - 1166: 4c 89 ef mov %r13,%rdi - 1169: 5b pop %rbx - 116a: 5d pop %rbp - 116b: 41 5c pop %r12 - 116d: 41 5d pop %r13 - 116f: 41 5e pop %r14 - 1171: 41 5f pop %r15 - 1173: e9 00 00 00 00 jmpq 1178 <build_dynamic_cell_key+0x178> - 1178: 0f 1f 84 00 00 00 00 nopl 0x0(%rax,%rax,1) - 117f: 00 - 1180: 48 8b 6c 24 08 mov 0x8(%rsp),%rbp - 1185: 48 8b 45 00 mov 0x0(%rbp),%rax - 1189: 48 89 c7 mov %rax,%rdi - 118c: 48 89 44 24 28 mov %rax,0x28(%rsp) - 1191: e8 00 00 00 00 callq 1196 <build_dynamic_cell_key+0x196> - 1196: 49 89 c7 mov %rax,%r15 - 1199: 48 8b 45 10 mov 0x10(%rbp),%rax - 119d: 48 89 c7 mov %rax,%rdi - 11a0: 48 89 44 24 30 mov %rax,0x30(%rsp) - 11a5: e8 00 00 00 00 callq 11aa <build_dynamic_cell_key+0x1aa> - 11aa: 48 89 44 24 38 mov %rax,0x38(%rsp) - 11af: 44 01 f8 add %r15d,%eax - 11b2: 48 63 e8 movslq %eax,%rbp - 11b5: 89 44 24 24 mov %eax,0x24(%rsp) - 11b9: 48 39 dd cmp %rbx,%rbp - 11bc: 72 24 jb 11e2 <build_dynamic_cell_key+0x1e2> - 11be: 4c 89 e8 mov %r13,%rax - 11c1: 0f 1f 80 00 00 00 00 nopl 0x0(%rax) - 11c8: 48 81 c3 00 04 00 00 add $0x400,%rbx - 11cf: 48 89 c7 mov %rax,%rdi - 11d2: 48 89 de mov %rbx,%rsi - 11d5: e8 00 00 00 00 callq 11da <build_dynamic_cell_key+0x1da> - 11da: 48 39 eb cmp %rbp,%rbx - 11dd: 76 e9 jbe 11c8 <build_dynamic_cell_key+0x1c8> - 11df: 49 89 c5 mov %rax,%r13 - 11e2: 48 8b 74 24 28 mov 0x28(%rsp),%rsi - 11e7: 4c 89 fa mov %r15,%rdx - 11ea: 4c 89 ef mov %r13,%rdi - 11ed: e8 00 00 00 00 callq 11f2 <build_dynamic_cell_key+0x1f2> - 11f2: 48 8b 54 24 38 mov 0x38(%rsp),%rdx - 11f7: 48 8b 74 24 30 mov 0x30(%rsp),%rsi - 11fc: 4b 8d 7c 3d 00 lea 0x0(%r13,%r15,1),%rdi - 1201: e8 00 00 00 00 callq 1206 <build_dynamic_cell_key+0x206> - 1206: e9 9c fe ff ff jmpq 10a7 <build_dynamic_cell_key+0xa7> - 120b: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) - 1210: 48 8b 44 24 08 mov 0x8(%rsp),%rax - 1215: 48 8b 00 mov (%rax),%rax - 1218: 48 89 c7 mov %rax,%rdi - 121b: 48 89 44 24 28 mov %rax,0x28(%rsp) - 1220: e8 00 00 00 00 callq 1225 <build_dynamic_cell_key+0x225> - 1225: 49 89 c7 mov %rax,%r15 - 1228: 8d 40 08 lea 0x8(%rax),%eax - 122b: 48 63 e8 movslq %eax,%rbp - 122e: 89 44 24 24 mov %eax,0x24(%rsp) - 1232: 48 39 dd cmp %rbx,%rbp - 1235: 72 23 jb 125a <build_dynamic_cell_key+0x25a> - 1237: 4c 89 e8 mov %r13,%rax - 123a: 66 0f 1f 44 00 00 nopw 0x0(%rax,%rax,1) - 1240: 48 81 c3 00 04 00 00 add $0x400,%rbx - 1247: 48 89 c7 mov %rax,%rdi - 124a: 48 89 de mov %rbx,%rsi - 124d: e8 00 00 00 00 callq 1252 <build_dynamic_cell_key+0x252> - 1252: 48 39 eb cmp %rbp,%rbx - 1255: 76 e9 jbe 1240 <build_dynamic_cell_key+0x240> - 1257: 49 89 c5 mov %rax,%r13 - 125a: 48 8b 74 24 28 mov 0x28(%rsp),%rsi - 125f: 4c 89 fa mov %r15,%rdx - 1262: 4c 89 ef mov %r13,%rdi - 1265: e8 00 00 00 00 callq 126a <build_dynamic_cell_key+0x26a> - 126a: 48 8b 44 24 08 mov 0x8(%rsp),%rax - 126f: 48 8b 40 10 mov 0x10(%rax),%rax - 1273: 4b 89 44 3d 00 mov %rax,0x0(%r13,%r15,1) - 1278: e9 2a fe ff ff jmpq 10a7 <build_dynamic_cell_key+0xa7> - 127d: 0f 1f 00 nopl (%rax) - 1280: 48 8b 44 24 08 mov 0x8(%rsp),%rax - 1285: 48 8b 00 mov (%rax),%rax - 1288: 48 89 c7 mov %rax,%rdi - 128b: 48 89 44 24 28 mov %rax,0x28(%rsp) - 1290: e8 00 00 00 00 callq 1295 <build_dynamic_cell_key+0x295> - 1295: 49 89 c7 mov %rax,%r15 - 1298: 8d 40 08 lea 0x8(%rax),%eax - 129b: 48 63 e8 movslq %eax,%rbp - 129e: 89 44 24 24 mov %eax,0x24(%rsp) - 12a2: 48 39 dd cmp %rbx,%rbp - 12a5: 72 b3 jb 125a <build_dynamic_cell_key+0x25a> - 12a7: 4c 89 e8 mov %r13,%rax - 12aa: 66 0f 1f 44 00 00 nopw 0x0(%rax,%rax,1) - 12b0: 48 81 c3 00 04 00 00 add $0x400,%rbx - 12b7: 48 89 c7 mov %rax,%rdi - 12ba: 48 89 de mov %rbx,%rsi - 12bd: e8 00 00 00 00 callq 12c2 <build_dynamic_cell_key+0x2c2> - 12c2: 48 39 eb cmp %rbp,%rbx - 12c5: 76 e9 jbe 12b0 <build_dynamic_cell_key+0x2b0> - 12c7: eb 8e jmp 1257 <build_dynamic_cell_key+0x257> - 12c9: 48 8d 3d 00 00 00 00 lea 0x0(%rip),%rdi # 12d0 <build_dynamic_cell_key+0x2d0> - 12d0: e8 00 00 00 00 callq 12d5 <build_dynamic_cell_key+0x2d5> - 12d5: 48 8b 54 24 48 mov 0x48(%rsp),%rdx - 12da: 48 89 c1 mov %rax,%rcx - 12dd: 48 89 02 mov %rax,(%rdx) - 12e0: 8b 31 mov (%rcx),%esi - 12e2: 48 83 c1 04 add $0x4,%rcx - 12e6: 8d 96 ff fe fe fe lea -0x1010101(%rsi),%edx - 12ec: f7 d6 not %esi - 12ee: 21 f2 and %esi,%edx - 12f0: 81 e2 80 80 80 80 and $0x80808080,%edx - 12f6: 74 e8 je 12e0 <build_dynamic_cell_key+0x2e0> - 12f8: 89 d6 mov %edx,%esi - 12fa: c1 ee 10 shr $0x10,%esi - 12fd: f7 c2 80 80 00 00 test $0x8080,%edx - 1303: 0f 44 d6 cmove %esi,%edx - 1306: 48 8d 71 02 lea 0x2(%rcx),%rsi - 130a: 48 0f 44 ce cmove %rsi,%rcx - 130e: 89 d6 mov %edx,%esi - 1310: 40 00 d6 add %dl,%sil - 1313: 48 83 d9 03 sbb $0x3,%rcx - 1317: 48 29 c1 sub %rax,%rcx - 131a: 48 8b 44 24 40 mov 0x40(%rsp),%rax - 131f: 48 89 08 mov %rcx,(%rax) - 1322: 48 83 c4 58 add $0x58,%rsp - 1326: 5b pop %rbx - 1327: 5d pop %rbp - 1328: 41 5c pop %r12 - 132a: 41 5d pop %r13 - 132c: 41 5e pop %r14 - 132e: 41 5f pop %r15 - 1330: c3 retq - 1331: bf 01 00 00 00 mov $0x1,%edi - 1336: 31 ed xor %ebp,%ebp - 1338: e9 f3 fd ff ff jmpq 1130 <build_dynamic_cell_key+0x130> - 133d: 0f 1f 00 nopl (%rax) - -0000000000001340 <tag_hash_key_get_compound_key>: - 1340: 48 83 ec 18 sub $0x18,%rsp - 1344: 48 8b 77 08 mov 0x8(%rdi),%rsi - 1348: 48 8b 3f mov (%rdi),%rdi - 134b: 48 8d 4c 24 08 lea 0x8(%rsp),%rcx - 1350: 48 89 e2 mov %rsp,%rdx - 1353: e8 00 00 00 00 callq 1358 <tag_hash_key_get_compound_key+0x18> - 1358: 48 8b 04 24 mov (%rsp),%rax - 135c: 48 83 c4 18 add $0x18,%rsp - 1360: c3 retq diff --git a/readme_fieldstat.md b/readme_fieldstat.md index cc17426..4dd7a99 100644 --- a/readme_fieldstat.md +++ b/readme_fieldstat.md @@ -40,9 +40,9 @@ Download fieldstat4 rpm from https://repo.geedge.net/pulp/content/ and install r #include "fieldstat.h" struct fieldstat *instance = fieldstat_new(); -int cube_id = fieldstat_create_cube(instance, YOUR_SHARED_TAG, YOUR_SHARED_TAG_LENGTH, SAMPLING_MODE_TOPK, MAX_CELL_NUMBER); +int cube_id = fieldstat_cube_create(instance, YOUR_SHARED_TAG, YOUR_SHARED_TAG_LENGTH, SAMPLING_MODE_TOPK, MAX_CELL_NUMBER); int metric_counter_id = fieldstat_register_counter(instance, cube_id, "any metric name", 0/1); -int metric_histogram_id = fieldstat_register_histogram(instance, cube_id, "any metric name", THE_MINIMUM_NUMBER_TO_RECORD, THE_MAXIMUM_NUMBER_TO_RECORD, PRECISION); +int metric_histogram_id = fieldstat_register_histogramogram(instance, cube_id, "any metric name", THE_MINIMUM_NUMBER_TO_RECORD, THE_MAXIMUM_NUMBER_TO_RECORD, PRECISION); int metric_hll_id = fieldstat_register_hll(instance, cube_id, "any metric name", PRECISION); int cell_id = fieldstat_cube_add(instance, cube_id, YOUR_TAG, YOUR_TAG_LENGTH, THE_PRIMARY_METRIC); if (cell_id != -1) { diff --git a/readme_fieldstat_easy.md b/readme_fieldstat_easy.md index 67e6901..c6f39cd 100644 --- a/readme_fieldstat_easy.md +++ b/readme_fieldstat_easy.md @@ -169,15 +169,15 @@ All the metrics in fieldstat easy are dynamic, which means that the metrics has #include "fieldstat_easy.h" const int N_THREADS = 3; -struct fieldstat_tag global_tags[1]; -struct fieldstat_tag tmptag; +struct field global_tags[1]; +struct field tmptag; tmptag.key = "app id"; -tmptag.type = TAG_INTEGER; +tmptag.type = FIELD_VALUE_INTEGER; tmptag.value_longlong = 1; global_tags[0] = tmptag; -struct fieldstat_tag tag1 = {"direction", TAG_STRING, "incoming"}; -struct fieldstat_tag tag2 = {"direction", TAG_STRING, "outgoing"}; +struct field tag1 = {"direction", TAG_STRING, "incoming"}; +struct field tag2 = {"direction", TAG_STRING, "outgoing"}; struct fieldstat_easy *fse = fieldstat_easy_new(N_THREADS, NULL, global_tags, 1); int counter_id = fieldstat_easy_register_counter(fse, "incoming bytes"); diff --git a/src/cells/hash_table.c b/src/cells/hash_table.c new file mode 100644 index 0000000..255a3ef --- /dev/null +++ b/src/cells/hash_table.c @@ -0,0 +1,233 @@ +#include "hash_table.h" + +#include <stdio.h> +#include <assert.h> +#include <stdlib.h> +#include <string.h> + +#include "uthash.h" + +#include "fieldstat.h" +#include "exdata.h" + +struct entry { + char *key; + size_t key_len; + void *exdata; + bool dying; + UT_hash_handle hh; +}; + +struct hash_table { + struct entry *key_exdata_map; + int current_cell_num; + int max_cell_num; + + exdata_new_cb new_fn; + exdata_free_cb free_fn; + exdata_merge_cb merge_fn; + exdata_reset_cb reset_fn; + exdata_copy_cb copy_fn; +}; +static void *default_new_fn(void *arg) { + return NULL; +} + +static void default_free_fn(void *exdata) { + return; +} + +static void default_merge_fn(void *dest, void *src) { + return; +} + +static void default_reset_fn(void *exdata) { + return; +} + +static void *default_copy_fn(void *exdata) { + return exdata; +} + + +struct hash_table *hash_table_new(int max_query_num) { + struct hash_table *pthis = calloc(1, sizeof(struct hash_table)); + + pthis->max_cell_num = max_query_num; + + pthis->new_fn = default_new_fn; + pthis->free_fn = default_free_fn; + pthis->merge_fn = default_merge_fn; + pthis->reset_fn = default_reset_fn; + pthis->copy_fn = default_copy_fn; + return pthis; +} + +void hash_table_free(struct hash_table *pthis) { + if (pthis == NULL) { + return; + } + struct entry *item, *tmp; + HASH_ITER(hh, pthis->key_exdata_map, item, tmp) { + HASH_DEL(pthis->key_exdata_map, item); + free(item->key); + pthis->free_fn(item->exdata); + free(item); + } + free(pthis); +} + +void hash_table_reset(struct hash_table *pthis) { + struct entry *node, *tmp; + HASH_ITER(hh, pthis->key_exdata_map, node, tmp) { + if (!node->dying) { + node->dying = true; + pthis->reset_fn(node->exdata); + continue; + } + HASH_DEL(pthis->key_exdata_map, node); + free(node->key); + pthis->free_fn(node->exdata); + free(node); + } + + pthis->current_cell_num = 0; +} + +static char *my_keydup(const char *key, size_t key_len) { + char *ret = calloc(1, key_len + 1); + memcpy(ret, key, key_len); + return ret; +} + +int hash_table_add(struct hash_table *pthis, const char *key, size_t key_len, void *arg) { + struct entry *item; + HASH_FIND(hh, pthis->key_exdata_map, key, key_len, item); + if (item != NULL && !item->dying) { + return 1; + } + + if (pthis->current_cell_num >= pthis->max_cell_num) { + return 0; + } + + if (item != NULL) { + assert(item->dying); + item->dying = false; + pthis->current_cell_num++; + return 1; + } + + item = calloc(1, sizeof(struct entry)); + item->key = my_keydup(key, key_len); + item->key_len = key_len; + item->exdata = pthis->new_fn(arg); + item->dying = false; + HASH_ADD_KEYPTR(hh, pthis->key_exdata_map, item->key, key_len, item); + + pthis->current_cell_num++; + return 1; +} + +void hash_table_set_exdata_schema(struct hash_table *pthis, exdata_new_cb new_fn, exdata_free_cb free_fn, exdata_merge_cb merge_fn, exdata_reset_cb reset_fn, exdata_copy_cb copy_fn) { + pthis->new_fn = new_fn; + pthis->free_fn = free_fn; + pthis->merge_fn = merge_fn; + pthis->reset_fn = reset_fn; + pthis->copy_fn = copy_fn; +} + +void *hash_table_get0_exdata(struct hash_table *pthis, const char *key, size_t key_len) { + struct entry *item; + HASH_FIND(hh, pthis->key_exdata_map, key, key_len, item); + if (item == NULL || item->dying) { + return NULL; + } + return item->exdata; +} + +int hash_table_get_count(const struct hash_table *pthis) { + return pthis->current_cell_num; +} + +size_t hash_table_list(const struct hash_table *pthis, void **exdatas, size_t n_exdatas) { + size_t actual_len = pthis->current_cell_num; + if (actual_len == 0) { + return 0; + } + + struct entry *item, *tmp; + size_t i = 0; + HASH_ITER(hh, pthis->key_exdata_map, item, tmp) { + if (item->dying) { + continue; + } + if (i >= n_exdatas) { + break; + } + exdatas[i] = item->exdata; + i++; + } + return actual_len < n_exdatas ? actual_len : n_exdatas; +} + +int hash_table_merge(struct hash_table *dest, struct hash_table *src) { + struct entry *item_src, *tmp; + struct entry *item_dst; + HASH_ITER(hh, src->key_exdata_map, item_src, tmp) { + if (item_src->dying) { + continue; + } + + HASH_FIND(hh, dest->key_exdata_map, item_src->key, item_src->key_len, item_dst); + if (item_dst != NULL && !item_dst->dying) { + dest->merge_fn(item_dst->exdata, item_src->exdata); + continue; + } + + if (dest->current_cell_num >= dest->max_cell_num) { + continue; // cannot add more cells + } + + if (item_dst == NULL) { + item_dst = calloc(1, sizeof(struct entry)); + item_dst->key = my_keydup(item_src->key, item_src->key_len); + item_dst->key_len = item_src->key_len; + item_dst->dying = false; + item_dst->exdata = dest->copy_fn(item_src->exdata); + HASH_ADD_KEYPTR(hh, dest->key_exdata_map, item_dst->key, item_dst->key_len, item_dst); + dest->current_cell_num++; + } else { + assert(item_dst->dying); + item_dst->dying = false; + dest->merge_fn(item_dst->exdata, item_src->exdata); //item_dst->exdata should be empty, but just merge to it. + dest->current_cell_num++; + } + } + return 0; +} + +struct hash_table *hash_table_copy(const struct hash_table *src) { + struct hash_table *pthis = calloc(1, sizeof(struct hash_table)); + pthis->max_cell_num = src->max_cell_num; + pthis->current_cell_num = src->current_cell_num; + pthis->new_fn = src->new_fn; + pthis->free_fn = src->free_fn; + pthis->merge_fn = src->merge_fn; + pthis->reset_fn = src->reset_fn; + pthis->copy_fn = src->copy_fn; + + struct entry *item, *tmp; + HASH_ITER(hh, src->key_exdata_map, item, tmp) { + if (item->dying) { + continue; + } + struct entry *new_item = calloc(1, sizeof(struct entry)); + new_item->key = my_keydup(item->key, item->key_len); + new_item->key_len = item->key_len; + new_item->exdata = pthis->copy_fn(item->exdata); + new_item->dying = false; + HASH_ADD_KEYPTR(hh, pthis->key_exdata_map, new_item->key, new_item->key_len, new_item); + } + return pthis; +} diff --git a/src/cells/hash_table.h b/src/cells/hash_table.h new file mode 100644 index 0000000..59fc8e8 --- /dev/null +++ b/src/cells/hash_table.h @@ -0,0 +1,30 @@ +#pragma once + +#include <stddef.h> + +#ifdef __cplusplus +extern "C"{ +#endif + +#include "exdata.h" + +struct hash_table; + +struct hash_table *hash_table_new(int max_query_num); +void hash_table_set_exdata_schema(struct hash_table *pthis, exdata_new_cb new_fn, exdata_free_cb free_fn, exdata_merge_cb merge_fn, exdata_reset_cb reset_fn, exdata_copy_cb copy_fn); +void hash_table_free(struct hash_table *pthis); +void hash_table_reset(struct hash_table *pthis); +int hash_table_merge(struct hash_table *dest, struct hash_table *src); +struct hash_table *hash_table_copy(const struct hash_table *src); + +// int hash_table_add(struct hash_table *pthis, const char *key, size_t key_len, int count, void *arg); +int hash_table_add(struct hash_table *pthis, const char *key, size_t key_len, void *arg); + +// void *hash_table_get0_exdata(struct hash_table *pthis, const char *key, size_t key_len); +void *hash_table_get0_exdata(struct hash_table *pthis, const char *key, size_t key_len); +int hash_table_get_count(const struct hash_table *pthis); +size_t hash_table_list(const struct hash_table *pthis, void **exdatas, size_t n_exdatas); + +#ifdef __cplusplus +} +#endif
\ No newline at end of file diff --git a/src/cells/heavy_keeper.c b/src/cells/heavy_keeper.c new file mode 100644 index 0000000..c62e550 --- /dev/null +++ b/src/cells/heavy_keeper.c @@ -0,0 +1,825 @@ +#include "heavy_keeper.h" + +#include <float.h> +#include <limits.h> +#include <math.h> +#include <stdbool.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#include "minheap/heap.h" +#include "mpack/mpack.h" +#include "xxhash/xxhash.h" +// XXHASH is faster +#define HASH_FUNCTION(keyptr, keylen, hashv) \ + do { \ + hashv = XXH3_64bits(keyptr, keylen); \ + } while (0) + +#include "uthash.h" + +#include "exdata.h" + +#define FP_HASH_KEY 0 +#define NOT_FIND (-1) +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) + +struct entry_data { // the value constitute of a sorted set entry + char *key; + size_t key_len; + + void *exdata; +}; + +struct sorted_set { + heap *heap; + struct hash_node *hash_tbl; + + int n_living_entry; + + exdata_new_cb new_fn; + exdata_free_cb free_fn; + exdata_merge_cb merge_fn; + exdata_reset_cb reset_fn; + exdata_copy_cb copy_fn; +}; + +struct hash_node { + const char *key; // value is equal to entry_data::key(point to the same memory) + size_t key_len; + heap_entry *val; + UT_hash_handle hh; +}; + +struct Bucket { + uint64_t finger_print; + long long count; // the actual count, sum of all uuid_hash_node->count +}; + +// Parameters used in algorithm +struct heavy_keeper_options{ + int r; // the size of the array. Default value: 4 + // Set it by how accurate you want. Value of 4 guarantees an accuracy more than 90% and around 97% in all tests. + // Not too big because it have an impact on both time and memory efficiency. + int w; // M2, the maximum number of buckets every array keeps. Default value: k*log(k) and no less than 100. + // Basically, as long as big enough, it won't affect the accuracy significantly. + double b; // b, bigger variance of flow size is(elephant flows take more ratio), smaller it should be. + // Must bigger than 1. Better not bigger than 1.3, otherwise some elephant flow will be missed. +}; + +struct heavy_keeper { + int K; + struct sorted_set *top_K_heap; + struct heavy_keeper_options params; + struct Bucket *sketch; + + unsigned int rand_state; + + exdata_new_cb new_fn; + exdata_free_cb free_fn; + exdata_merge_cb merge_fn; + exdata_reset_cb reset_fn; + exdata_copy_cb copy_fn; +}; + +static void *default_new_fn(void *arg) { + return NULL; +} + +static void default_free_fn(void *exdata) { + return; +} + +static void default_merge_fn(void *dest, void *src) { + return; +} + +static void default_reset_fn(void *exdata) { + return; +} + +static void *default_copy_fn(void *exdata) { + return exdata; +} + +struct entry_data *entry_data_construct(const char *key, size_t key_len, void *exdata) +{ + struct entry_data *entry_data = (struct entry_data *)malloc(sizeof(struct entry_data)); + + entry_data->key = (char *)malloc(key_len); + memcpy(entry_data->key, key, key_len); + entry_data->key_len = key_len; + entry_data->exdata = exdata; + return entry_data; +} + +void entry_data_destroy(struct entry_data *entry_data, exdata_free_cb free_fn) +{ + if (entry_data == NULL) { + return; + } + free(entry_data->key); + free_fn(entry_data->exdata); + free(entry_data); +} + +static inline long long sorted_set_entry_get_score(const heap_entry *entry) +{ + long long score = *(long long *)entry->key; + if (score == 0) { + return 0; + } + return score - 1; // sorted set will let the count start from 1, 0 for dying entry. +} + +struct entry_data *sorted_set_entry_get_data(const heap_entry *entry) +{ + return entry->value; +} + +/* + dying: To reduce the time of HASH_ADD, and to support merely recording a key without any value, give a dying status. +*/ +static inline bool sorted_set_entry_dying(const heap_entry *entry) +{ + long long score = *(long long *)entry->key; + return score == 0; +} + +static inline long long safe_add(long long a, long long b) +{ + if (INT64_MAX - a < b) { + return INT64_MAX; + } + return a + b; +} + +// cppcheck-suppress [constParameterCallback, unmatchedSuppression] +static bool cmp_entry_cb(void *aa, void *bb) +{ + heap_entry *a = (heap_entry *)aa; + heap_entry *b = (heap_entry *)bb; + long long score_a = *(long long *)a->key; + long long score_b = *(long long *)b->key; + + if (score_a < score_b) { + return true; + } + return false; +} + +static int entry_get_index_cb(void *a) +{ + return ((heap_entry *)a)->index; +} + +void entry_set_index_cb(int index, void *a) +{ + ((heap_entry *)a)->index = index; +} + +struct sorted_set *sorted_set_new(int set_size) +{ + struct sorted_set *ret = (struct sorted_set *)malloc(sizeof(struct sorted_set)); + ret->hash_tbl = NULL; + ret->n_living_entry = 0; + heap *h = (heap *)malloc(sizeof(heap)); + init_heap(h, set_size, cmp_entry_cb, entry_get_index_cb, entry_set_index_cb); + + ret->heap = h; + + ret->new_fn = default_new_fn; + ret->free_fn = default_free_fn; + ret->merge_fn = default_merge_fn; + ret->reset_fn = default_reset_fn; + ret->copy_fn = default_copy_fn; + + return ret; +} + +void sorted_set_schema_set(struct sorted_set *ss, exdata_new_cb new_fn, exdata_free_cb free_fn, exdata_merge_cb merge_fn, exdata_reset_cb reset_fn, exdata_copy_cb copy_fn) +{ + ss->new_fn = new_fn; + ss->free_fn = free_fn; + ss->merge_fn = merge_fn; + ss->reset_fn = reset_fn; + ss->copy_fn = copy_fn; +} + +void sorted_set_free(struct sorted_set *ss) +{ + struct hash_node *tbl = ss->hash_tbl; + struct hash_node *h_node, *tmp; + HASH_ITER(hh, tbl, h_node, tmp) { + HASH_DEL(tbl, h_node); + free(h_node); + } + + heap *heap = ss->heap; + heap_entry *node; + for (int i = 0; i < heap->cur_size; i++) { + node = (heap_entry *)heap->nodes[i]; + entry_data_destroy(node->value, ss->free_fn); + free(node->key); + free(node); + } + free(heap->nodes); + free(heap); + + free(ss); +} + +heap_entry *sorted_set_find_entry(const struct sorted_set *ss, const char *key, size_t key_len) +{ + struct hash_node *hash_tbl = ss->hash_tbl; + struct hash_node *s = NULL; + HASH_FIND(hh, hash_tbl, key, key_len, s); + + if (s == NULL) { + return NULL; + } + return (heap_entry *)s->val; +} + +static inline void sorted_set_entry_set_score(struct sorted_set *ss, heap_entry *entry, long long score) +{ + *(long long *)(entry->key) = score; + + adjust_heap_node(ss->heap, entry); +} + +int sorted_set_pop(struct sorted_set *ss) +{ + struct hash_node *hash_tbl = ss->hash_tbl; + heap_entry *entry = (heap_entry *)pop_heap(ss->heap); + if (entry == NULL) { + return -1; + } + struct entry_data *tmp_data = sorted_set_entry_get_data(entry); + + struct hash_node *s = NULL; + HASH_FIND(hh, hash_tbl, tmp_data->key, tmp_data->key_len, s); + assert(s!=NULL); + + HASH_DEL(hash_tbl, s); + free(s); + ss->hash_tbl = hash_tbl; + + int ret; + if (sorted_set_entry_dying(entry)) { + ret = 0; + } else { + ss->n_living_entry--; + ret = 1; + } + + entry_data_destroy(tmp_data, ss->free_fn); + free(entry->key); + free(entry); + + return ret; +} + +static inline bool sorted_set_check_is_full(const struct sorted_set *ss) +{ + return ss->heap->cur_size >= ss->heap->max_size; +} + +long long sorted_set_get_min_score(const struct sorted_set *ss) +{ + heap *heap = ss->heap; + if (heap->cur_size == 0) { + return NOT_FIND; + } + const heap_entry *ret = (heap_entry *)(heap->nodes[0]); + + long long score = *(long long *)ret->key; + if (score == 0) { + return NOT_FIND; + } + return score - 1; // sorted set will let the score start from 1, 0 for dying entry. +} + +void sorted_set_insert_to_available_heap(struct sorted_set *ss, const char *key, size_t key_len, long long cnt, void *exdata) +{ + cnt = safe_add(cnt, 1); // sorted set will let the score start from 1, 0 for dying entry. + long long *tmp_cnt = (long long*)malloc(sizeof(long long)); + *tmp_cnt = cnt; + struct entry_data *tmp_data = entry_data_construct(key, key_len, exdata); + heap_entry *node = (heap_entry *)malloc(sizeof(heap_entry)); + node->key = tmp_cnt; + node->value = tmp_data; + + push_heap(ss->heap, (void *)node); + + struct hash_node *hash_tbl = ss->hash_tbl; + struct hash_node *s = (struct hash_node *)malloc(sizeof(struct hash_node)); + s->key = sorted_set_entry_get_data(node)->key; + s->key_len = key_len; + s->val = node; + HASH_ADD_KEYPTR(hh, hash_tbl, s->key, key_len, s); + ss->hash_tbl = hash_tbl; + + ss->n_living_entry++; +} + +int sorted_set_insert(struct sorted_set *ss, const char *key, size_t key_len, long long cnt, void *args) +{ + // if there is a dying record, reborn it to use. + heap_entry *entry = sorted_set_find_entry(ss, key, key_len); + if (entry != NULL) { + if (!sorted_set_entry_dying(entry)) { + assert(0); + return -1; + } + sorted_set_entry_set_score(ss, entry, safe_add(cnt, 1)); // sorted set will let the score start from 1, 0 for dying entry. + + ss->n_living_entry++; + return 1; + } + + if (sorted_set_check_is_full(ss)) { + long long tmp_mincnt = sorted_set_get_min_score(ss); + if (tmp_mincnt != NOT_FIND && cnt <= tmp_mincnt) { // even if all cells in sorted set are dying, the sorted set can still be a full one, in which case, the min score is invalid. + return 0; + } + sorted_set_pop(ss); + } + sorted_set_insert_to_available_heap(ss, key, key_len, cnt, ss->new_fn(args)); + return 1; +} + +static inline int sorted_set_cardinality(const struct sorted_set *ss) +{ + return ss->n_living_entry; +} + +long long sorted_set_get_score(const struct sorted_set *ss, const char *key, size_t key_len) +{ + if (sorted_set_cardinality(ss) == 0) { + return NOT_FIND; + } + const heap_entry *entry = sorted_set_find_entry(ss, key, key_len); + if (entry == NULL) { + return NOT_FIND; + } + if (sorted_set_entry_dying(entry)) { + return NOT_FIND; + } + + return sorted_set_entry_get_score(entry); +} + +int sorted_set_incrby(struct sorted_set *ss, const char *key, size_t key_len, long long score) +{ + heap_entry *entry = sorted_set_find_entry(ss, key, key_len); + if (entry == NULL) { + return -1; + } + long long cnt_old; + if (sorted_set_entry_dying(entry) == false) { + cnt_old = sorted_set_entry_get_score(entry); + cnt_old += 1; // sorted set will let the score start from 1, 0 for dying entry. + } else { + cnt_old = 0; + } + sorted_set_entry_set_score(ss, entry, safe_add(score, cnt_old)); + return 0; +} + +typedef struct { + long long key; + struct entry_data *val; +} tmp_heap_node; + +int cmp_tmp_heap_node(const void *a, const void *b) +{ + const tmp_heap_node *aa = (tmp_heap_node *)a; + const tmp_heap_node *bb = (tmp_heap_node *)b; + + if (aa->key < bb->key) { + return 1; + } else if (aa->key > bb->key) { + return -1; + } else { + return 0; + } +} + +void sorted_set_dump(const struct sorted_set *ss, char **key_out, size_t *key_len_out, void **exdata_out) +{ + struct minheap *h = ss->heap; + tmp_heap_node *tmp_nodes = (tmp_heap_node *)malloc(sizeof(tmp_heap_node) * h->cur_size); + size_t n_living_entry = 0; + for (int i = 0; i < h->cur_size; i++) { + const heap_entry *entry = (heap_entry *)h->nodes[i]; + if (sorted_set_entry_dying(entry)) { + continue; + } + tmp_nodes[n_living_entry].key = sorted_set_entry_get_score(entry); + tmp_nodes[n_living_entry].val = sorted_set_entry_get_data(entry); + n_living_entry++; + } + + assert(n_living_entry == ss->n_living_entry); + qsort(tmp_nodes, n_living_entry, sizeof(tmp_heap_node), cmp_tmp_heap_node); + for (int i = 0; i < n_living_entry; i++) { + key_out[i] = tmp_nodes[i].val->key; + key_len_out[i] = tmp_nodes[i].val->key_len; + exdata_out[i] = tmp_nodes[i].val->exdata; + } + free(tmp_nodes); +} + +struct sorted_set *sorted_set_copy(const struct sorted_set *ss) +{ + struct sorted_set *ret = sorted_set_new(ss->heap->max_size); + sorted_set_schema_set(ret, ss->new_fn, ss->free_fn, ss->merge_fn, ss->reset_fn, ss->copy_fn); + + for (int i = 0; i < ss->heap->cur_size; i++) { + const heap_entry *entry = (heap_entry *)ss->heap->nodes[i]; + if (sorted_set_entry_dying(entry)) { + continue; + } + const struct entry_data *data = sorted_set_entry_get_data(entry); + sorted_set_insert_to_available_heap(ret, data->key, data->key_len, sorted_set_entry_get_score(entry), ss->copy_fn(data->exdata)); + } + + return ret; +} + +// set all the entry to dying status. They will be the first to be popped. +void sorted_set_reset(struct sorted_set *ss) +{ + heap *heap = ss->heap; + heap_entry *entry; + for (int i = 0; i < heap->cur_size; i++) { + entry = (heap_entry *)heap->nodes[i]; + *(long long *)entry->key = 0; + + ss->reset_fn(sorted_set_entry_get_data(entry)->exdata); + } + + ss->n_living_entry = 0; +} + +/* -------------------------------------------------------------------------- */ +/* heavy keeper */ +/* -------------------------------------------------------------------------- */ + +struct Bucket *new_sketch(struct heavy_keeper_options *params) { + size_t array_len = (size_t)params->r * (size_t)params->w; + + struct Bucket *ret = (struct Bucket *)calloc(array_len, sizeof(struct Bucket)); + + return ret; +} + +void params_set_to_default(struct heavy_keeper_options *p, int K) { + if (K > 5000) { // don't let the sketch too large when K gets insanely large + K = 5000; + } + double log_ret = log((double)K) / 2.3; // 2.3: log_e(10), log_ret = log_10(K) + + if (log_ret < 3) { + p->r = 3; + } else { + p->r = (int)(log_ret); + } + p->b = 1.17; // by test, 1.17 is the best + p->w = (int)(log_ret * K * 2); + if (p->w < 150) { + p->w = 150; // determined through test, too small max_bucket_num will let accuracy decrease severely. + } else if (p->w > 600) { + p->w = p->w / 4 + 450; + } +} + +struct heavy_keeper *heavy_keeper_new(int max_query_num) { + struct heavy_keeper *hk = (struct heavy_keeper *)malloc(sizeof(struct heavy_keeper)); + hk->K = max_query_num; + hk->rand_state = 0; + + hk->top_K_heap = sorted_set_new(max_query_num); + + params_set_to_default(&(hk->params), max_query_num); + + hk->sketch = new_sketch(&(hk->params)); + + hk->new_fn = default_new_fn; + hk->free_fn = default_free_fn; + hk->merge_fn = default_merge_fn; + hk->reset_fn = default_reset_fn; + + return hk; +} + +void heavy_keeper_free(struct heavy_keeper *hk) { + if (hk == NULL) { + return; + } + sorted_set_free(hk->top_K_heap); + free(hk->sketch); + + free(hk); +} + +void heavy_keeper_reset(struct heavy_keeper *hk) { + sorted_set_reset(hk->top_K_heap); + memset(hk->sketch, 0, sizeof(struct Bucket) * (size_t)hk->params.r * (size_t)hk->params.w); +} + +const int DECAY_POW_TABLE[128] = { // 1.17 ^ exp * RAND_MAX, exp is in [0,127] +2147483647,1835456109,1568765905,1340825560,1146004752,979491241,837172001,715531625,611565491,522705548,446756879,381843486,326361954,278941841,238411830,203770795,174163072,148857327,127228484,108742294,92942132, +79437720,67895487,58030331,49598573,42391943,36232430,30967889,26468281,22622462,19335438,16526015,14124799,12072478,10318357,8819109,7537700,6442479,5506392,4706318,4022494,3438029,2938486,2511527,2146604, +1834704,1568123,1340276,1145535,979090,836829,715239,611315,522491,446574,381687,326228,278828,238314,203687,174092,148796,127176,108698,92904,79405,67868,58007,49578,42375,36218,30955,26457,22613,19328,16519, +14119,12068,10314,8815,7535,6440,5504,4704,4021,3437,2937,2510,2146,1834,1567,1340,1145,979,836,715,611,522,446,382,326,279,238,204,174,149,127,109,93,79,68,58,50,42,36,31,26,23,19,17,14,12,10,9,8,6,6,5, +}; +bool if_need_to_decay(struct heavy_keeper *hk, const struct Bucket *bucket, long long count) { + if (count == 0) { + return false; + } + if (bucket->count < count) { // the exp is 0, so the decay rate is 1 + return true; + } + + long long exp = bucket->count / count; + if (exp > 127) { // decay_rate too small, almost no chance to decay + return false; + } + + // double decay_rate = pow(hk->params.b, -exp); + // p->b = 1.17 is fixed, search table to get result directly. + int decay_rate = DECAY_POW_TABLE[exp]; + + if (rand_r(&(hk->rand_state)) < decay_rate) { + return true; + } + return false; +} + +static inline uint64_t cal_hash_val_with_seed(const char *key, size_t key_len, unsigned int seed) { + return XXH3_64bits_withSeed(key, key_len, seed); +} + +/* +1 for newly add something. 0 for not add. -1 for unexpected cases. +*/ +int heavy_keeper_add(struct heavy_keeper *heavy_keeper, const char *key, size_t key_len, long long count, void *arg) { + assert(count >= 0); + if (count == 0) { + if (sorted_set_cardinality(heavy_keeper->top_K_heap) < heavy_keeper->K) { + const struct heap_entry *entry = sorted_set_find_entry(heavy_keeper->top_K_heap, key, key_len); + + if (entry != NULL && !sorted_set_entry_dying(entry)) { + return 0; + } + + sorted_set_insert(heavy_keeper->top_K_heap, key, key_len, count, arg); + return 1; + } + return 0; + } + + struct sorted_set *summary = heavy_keeper->top_K_heap; + + long long old_cnt = sorted_set_get_score(summary, key, key_len); + bool not_in_sorted_set = (old_cnt == NOT_FIND); + long long maxv = 0; + uint64_t fp = cal_hash_val_with_seed(key, key_len, FP_HASH_KEY); + uint64_t h1 = fp; + uint64_t h2 = cal_hash_val_with_seed(key, key_len, FP_HASH_KEY+1); + + for (uint64_t j = 0; j < heavy_keeper->params.r; j++) { + uint64_t hashv = h1 + j * h2; // use `double hashing` strategy + struct Bucket *bucket = &(heavy_keeper->sketch[j * heavy_keeper->params.w + (hashv % heavy_keeper->params.w)]); + + if (bucket->finger_print == fp) { + // If a key is not in the min-heap, then the estimated key size should be no larger than nmin. + // or if the min-heap is not full(min_value == NOT_FIND), every key should be taken into account, so of course it should be added. + // in neither case, bucket->count > nMin && not_in_sorted_set happen. + // The keys whose counts are both larger than nmin and not in min-heap must have the same xxhash value, and its FP stored in bucket represents another field, + // In this case, the sketch won't be updated. This key is expected to be taken into account in another array, + // where its FP is different from the one it should collided with, so that element keys won't be missed. + if (not_in_sorted_set) { + long long min_value = sorted_set_get_min_score(summary); + if (min_value != NOT_FIND && bucket->count > min_value) { + continue; + } + } + bucket->count = safe_add(bucket->count, count); + maxv = MAX(maxv, bucket->count); + } else { + if (!if_need_to_decay(heavy_keeper, bucket, count)) { + continue; + } + + if (bucket->count < count) { + bucket->finger_print = fp; + bucket->count = count; + + maxv = MAX(maxv, count); + } else { + bucket->count -= count; + } + } + } + + if (not_in_sorted_set) { + if (sorted_set_cardinality(summary) != heavy_keeper->K) { + sorted_set_insert(summary, key, key_len, maxv, arg); + return 1; + } + long long min_value = sorted_set_get_min_score(summary); + if (maxv > min_value || min_value == NOT_FIND) { + sorted_set_insert(summary, key, key_len, maxv, arg); + return 1; + } + return 0; + } else { + if (maxv > old_cnt) { + sorted_set_incrby(summary, key, key_len, maxv - old_cnt); + } + return 1; // no popped, but the exdata definitely exists in the sorted set + } +} + +int heavy_keeper_set_exdata_schema(struct heavy_keeper *hk, exdata_new_cb new_fn, exdata_free_cb free_fn, exdata_merge_cb merge_fn, exdata_reset_cb reset_fn, exdata_copy_cb copy_fn) { + sorted_set_schema_set(hk->top_K_heap, new_fn, free_fn, merge_fn, reset_fn, copy_fn); + hk->new_fn = new_fn; + hk->free_fn = free_fn; + hk->merge_fn = merge_fn; + hk->reset_fn = reset_fn; + hk->copy_fn = copy_fn; + + return 0; +} + +void *heavy_keeper_get0_exdata(const struct heavy_keeper *hk, const char *key, size_t key_len) { + const heap_entry *entry = sorted_set_find_entry(hk->top_K_heap, key, key_len); + if (entry == NULL || sorted_set_entry_dying(entry)) { + return NULL; + } + return sorted_set_entry_get_data(entry)->exdata; +} + +int heavy_keeper_get_count(const struct heavy_keeper *hk) { + return sorted_set_cardinality(hk->top_K_heap); +} + +size_t heavy_keeper_list(const struct heavy_keeper *hk, void **exdatas, size_t n_exdatas) { + size_t actual_len = sorted_set_cardinality(hk->top_K_heap); + if (actual_len == 0) { + return 0; + } + + char **keys_dummy = (char **)malloc(sizeof(char *) * actual_len); + size_t *key_lens_dummy = (size_t *)malloc(sizeof(size_t) * actual_len); + if (n_exdatas < actual_len) { + void **exdatas_ret = (void **)malloc(sizeof(void *) * actual_len); + sorted_set_dump(hk->top_K_heap, keys_dummy, key_lens_dummy, exdatas_ret); + memcpy(exdatas, exdatas_ret, sizeof(void *) * n_exdatas); + free(exdatas_ret); + } else { + sorted_set_dump(hk->top_K_heap, keys_dummy, key_lens_dummy, exdatas); + } + + free(keys_dummy); + free(key_lens_dummy); + + return actual_len < n_exdatas ? actual_len : n_exdatas; +} + +static void heavy_keeper_merge_sketch(struct heavy_keeper *dest, const struct heavy_keeper *src) { + int w = dest->params.w; + int d = dest->params.r; + //idx + for (int array_id = 0; array_id < d; array_id++) { + for (int bucket_id = 0; bucket_id < w; bucket_id++) { + struct Bucket *bucket_dest = &(dest->sketch[array_id * w + bucket_id]); + const struct Bucket *bucket_src = &(src->sketch[array_id * w + bucket_id]); + + if (bucket_dest->finger_print == bucket_src->finger_print) { + bucket_dest->count = safe_add(bucket_dest->count, bucket_src->count); + } else { + if (bucket_dest->count < bucket_src->count) { + bucket_dest->count = bucket_src->count; + bucket_dest->finger_print = bucket_src->finger_print; + } + } + } + } +} + +long long find_count_in_sketch(struct heavy_keeper *hk, const char *key, size_t key_len) { + struct Bucket *bucket; + uint64_t fp = cal_hash_val_with_seed(key, key_len, FP_HASH_KEY); + uint64_t h1 = fp; + uint64_t h2 = cal_hash_val_with_seed(key, key_len, FP_HASH_KEY+1); + + long long maxv = 0; + for (uint64_t array_id = 0; array_id < hk->params.r; array_id++) { + uint64_t hash = h1 + array_id * h2; + bucket = &(hk->sketch[array_id * hk->params.w + (hash % hk->params.w)]); + + if (bucket->finger_print == fp) { + maxv = MAX(maxv, bucket->count); + } + } + + return maxv; +} + +void heavy_keeper_merge(struct heavy_keeper *dest, const struct heavy_keeper *src) { + assert(dest->K == src->K); + + heavy_keeper_merge_sketch(dest, src); + + struct sorted_set *new_rec = sorted_set_new(dest->K); // merging result + sorted_set_schema_set(new_rec, dest->new_fn, dest->free_fn, dest->merge_fn, dest->reset_fn, dest->copy_fn); + const struct sorted_set *ss_dest = dest->top_K_heap; // original ss in dest + const struct sorted_set *ss_src = src->top_K_heap; // the ss to be merged + int size_dest = sorted_set_cardinality(ss_dest); + int size_src = sorted_set_cardinality(ss_src); + int max_size = size_dest > size_src ? size_dest : size_src; + + void **exdatas_dst = (void **)calloc(size_dest, sizeof(void *)); + char **key_arr = (char **)malloc(sizeof(char *) * max_size); + size_t *key_lens = (size_t *)malloc(sizeof(size_t) * max_size); + sorted_set_dump(ss_dest, key_arr, key_lens, exdatas_dst); + + + /* ------------------------------ merge dest ------------------------------ */ + for (int i = 0; i < size_dest; i++) { + long long maxv = find_count_in_sketch(dest, key_arr[i], key_lens[i]); + sorted_set_insert_to_available_heap(new_rec, key_arr[i], key_lens[i], maxv, dest->copy_fn(exdatas_dst[i])); + } + + /* ------------------------------ merge src ------------------------------ */ + void **exdatas_src = (void **)calloc(size_src, sizeof(void *)); + sorted_set_dump(ss_src, key_arr, key_lens, exdatas_src); + + for (int i = 0; i < size_src; i++) { + const heap_entry *entry = sorted_set_find_entry(new_rec, key_arr[i], key_lens[i]); + if (entry != NULL) { // the key is in both dest and src, so has been processed in the previous loop. The reason why no need to sum up result is that merged sketch already keeps its summed up count + void *exdata_new = sorted_set_entry_get_data(entry)->exdata; + void *exdata_src = exdatas_src[i]; + dest->merge_fn(exdata_new, exdata_src); + + continue; + } + + long long cnt = find_count_in_sketch(dest, key_arr[i], key_lens[i]);// the cnt is the estimated count in the merged sketch, since the dest heavy keeper has been merged + if (sorted_set_check_is_full(new_rec)) { + long long mincnt_new = sorted_set_get_min_score(new_rec); + if (cnt > mincnt_new) { + sorted_set_pop(new_rec); + sorted_set_insert_to_available_heap(new_rec, key_arr[i], key_lens[i], cnt, dest->copy_fn(exdatas_src[i])); + } + } else { + sorted_set_insert_to_available_heap(new_rec, key_arr[i], key_lens[i], cnt, dest->copy_fn(exdatas_src[i])); + } + } + + free(key_arr); + free(key_lens); + free(exdatas_dst); + free(exdatas_src); + sorted_set_free(dest->top_K_heap); + dest->top_K_heap = new_rec; +} + +struct heavy_keeper *heavy_keeper_copy(const struct heavy_keeper *src) { + struct heavy_keeper *ret = (struct heavy_keeper *)malloc(sizeof(struct heavy_keeper)); + ret->K = src->K; + ret->rand_state = 0; + + ret->top_K_heap = sorted_set_copy(src->top_K_heap); + + ret->params = src->params; + ret->sketch = (struct Bucket *)malloc(sizeof(struct Bucket) * (size_t)ret->params.r * (size_t)ret->params.w); + memcpy(ret->sketch, src->sketch, sizeof(struct Bucket) * (size_t)ret->params.r * (size_t)ret->params.w); + + ret->new_fn = src->new_fn; + ret->free_fn = src->free_fn; + ret->merge_fn = src->merge_fn; + ret->reset_fn = src->reset_fn; + ret->copy_fn = src->copy_fn; + + return ret; +} + +void heavy_keeper_one_point_query(const struct heavy_keeper *hk, const char *key, size_t key_len, long long *count_out, void **exdata_out) { + *count_out = 0; + *exdata_out = NULL; + + const heap_entry *entry = sorted_set_find_entry(hk->top_K_heap, key, key_len); + if (entry == NULL) { + return; + } + *count_out = sorted_set_entry_get_score(entry); + *exdata_out = sorted_set_entry_get_data(entry)->exdata; +}
\ No newline at end of file diff --git a/src/cells/heavy_keeper.h b/src/cells/heavy_keeper.h new file mode 100644 index 0000000..3b09598 --- /dev/null +++ b/src/cells/heavy_keeper.h @@ -0,0 +1,48 @@ +#pragma once + +#include <stddef.h> + +#ifdef __cplusplus +extern "C"{ +#endif + +#include "exdata.h" + + +struct heavy_keeper; + +struct heavy_keeper *heavy_keeper_new(int max_query_num); +/** + * @brief free a heavy keeper. + * @param hk the pointer to the heavy keeper. + */ +void heavy_keeper_free(struct heavy_keeper *hk); + +/* + * @brief clear a heavy keeper. It will clear all the data in the heavy keeper. as for every exdata, use the reset function `exdata_reset_cb` to reset it. + * @param hk the pointer to the heavy keeper. +*/ +void heavy_keeper_reset(struct heavy_keeper *hk); + +int heavy_keeper_add(struct heavy_keeper *hk, const char *key, size_t key_len, long long count, void *arg); + +int heavy_keeper_set_exdata_schema(struct heavy_keeper *hk, exdata_new_cb new_fn, exdata_free_cb free_fn, exdata_merge_cb merge_fn, exdata_reset_cb reset_fn, exdata_copy_cb copy_fn); + +void *heavy_keeper_get0_exdata(const struct heavy_keeper *hk, const char *key, size_t key_len); + +// get the number of cells in the heavy keeper +int heavy_keeper_get_count(const struct heavy_keeper *hk); + +size_t heavy_keeper_list(const struct heavy_keeper *hk, void **exdatas, size_t n_exdatas); + +void heavy_keeper_merge(struct heavy_keeper *dest, const struct heavy_keeper *src); + +struct heavy_keeper *heavy_keeper_copy(const struct heavy_keeper *src); + +// for test +void heavy_keeper_one_point_query(const struct heavy_keeper *hk, const char *key, size_t key_len, long long *count_out, void **exdata_out); + + +#ifdef __cplusplus +} +#endif
\ No newline at end of file diff --git a/src/cells/hll_common.c b/src/cells/hll_common.c new file mode 100644 index 0000000..47dc113 --- /dev/null +++ b/src/cells/hll_common.c @@ -0,0 +1,384 @@ + +#include "hll_common.h" +#include "crdt_utils.h" + +#include "base64/b64.h" + +#include <string.h> +#include <math.h> +#include <assert.h> +#define REG_WIDTH 6 // Bits per register +#define INT_WIDTH 32 // Bits in an int +#define REG_PER_WORD 5 // floor(INT_WIDTH / REG_WIDTH) + +#define NUM_REG(precision) ((1 << precision)) +#define INT_CEIL(num, denom) (((num) + (denom) - 1) / (denom)) + +#define PRECISION_TO_WORD(precision) INT_CEIL(NUM_REG(precision), REG_PER_WORD) +#define RESET_TIME_SLOT_US(time_window_ms, precision) (time_window_ms*2*1000/NUM_REG(precision)) +#define REGISTER_SIZE(precision) INT_WIDTH*INT_CEIL(NUM_REG(precision), REG_PER_WORD) + +static int get_register(const uint32_t *registers, int idx) { + uint32_t word = *(registers + (idx / REG_PER_WORD)); + word = word >> REG_WIDTH * (idx % REG_PER_WORD); + return word & ((1 << REG_WIDTH) - 1); +} + +static void set_register(uint32_t *registers, int idx, int val) { + uint32_t *word = registers + (idx / REG_PER_WORD); + + // Shift the val into place + unsigned shift = REG_WIDTH * (idx % REG_PER_WORD); + val = val << shift; + uint32_t val_mask = ((1 << REG_WIDTH) - 1) << shift; + + // Store the word + *word = (*word & ~val_mask) | val; + return; +} +static void reset_register(uint32_t *registers, int idx) +{ + uint32_t *word = registers + (idx / REG_PER_WORD); + unsigned shift = REG_WIDTH * (idx % REG_PER_WORD); + uint32_t val_mask = ((1 << REG_WIDTH) - 1) << shift; + *word &= ~val_mask; +} +uint32_t *hll_create_register(int precision) +{ + return ALLOC(uint32_t, PRECISION_TO_WORD(precision)); +} +uint32_t *hll_duplicate(const uint32_t *registers, unsigned char precision) +{ + int words = PRECISION_TO_WORD(precision); + uint32_t *new_registers = ALLOC(uint32_t, words); + memcpy(new_registers, registers, sizeof(uint32_t)*words); + return new_registers; +} +void hll_free_register(uint32_t *registers) +{ + free(registers); + return; +} +static void clear_register(uint32_t *registers, int precision) +{ + memset(registers, 0, PRECISION_TO_WORD(precision)*sizeof(uint32_t)); + return; +} +int hll_add_hash(uint32_t *registers, unsigned char precision, uint64_t hash) +{ + // Determine the index using the first p bits + int idx = hash >> (64 - precision); + + // Shift out the index bits + hash = hash << precision | (1 << (precision -1)); + + // Determine the count of leading zeros. The __builtin_clzll() is provided by GCC + int leading = __builtin_clzll(hash) + 1; + + // Update the register if the new value is larger + if (leading > get_register(registers, idx)) { + set_register(registers, idx, leading); + return 1; + } + return 0; +} + + +static const double *g_rawEstimateData[] = { + // precision 4 + (double[]) { 11, 11.717, 12.207, 12.7896, 13.2882, 13.8204, 14.3772, 14.9342, 15.5202, 16.161, 16.7722, 17.4636, 18.0396, 18.6766, 19.3566, 20.0454, 20.7936, 21.4856, 22.2666, 22.9946, 23.766, 24.4692, 25.3638, 26.0764, 26.7864, 27.7602, 28.4814, 29.433, 30.2926, 31.0664, 31.9996, 32.7956, 33.5366, 34.5894, 35.5738, 36.2698, 37.3682, 38.0544, 39.2342, 40.0108, 40.7966, 41.9298, 42.8704, 43.6358, 44.5194, 45.773, 46.6772, 47.6174, 48.4888, 49.3304, 50.2506, 51.4996, 52.3824, 53.3078, 54.3984, 55.5838, 56.6618, 57.2174, 58.3514, 59.0802, 60.1482, 61.0376, 62.3598, 62.8078, 63.9744, 64.914, 65.781, 67.1806, 68.0594, 68.8446, 69.7928, 70.8248, 71.8324, 72.8598, 73.6246, 74.7014, 75.393, 76.6708, 77.2394, }, + // precision 5 + (double[]) { 23, 23.1194, 23.8208, 24.2318, 24.77, 25.2436, 25.7774, 26.2848, 26.8224, 27.3742, 27.9336, 28.503, 29.0494, 29.6292, 30.2124, 30.798, 31.367, 31.9728, 32.5944, 33.217, 33.8438, 34.3696, 35.0956, 35.7044, 36.324, 37.0668, 37.6698, 38.3644, 39.049, 39.6918, 40.4146, 41.082, 41.687, 42.5398, 43.2462, 43.857, 44.6606, 45.4168, 46.1248, 46.9222, 47.6804, 48.447, 49.3454, 49.9594, 50.7636, 51.5776, 52.331, 53.19, 53.9676, 54.7564, 55.5314, 56.4442, 57.3708, 57.9774, 58.9624, 59.8796, 60.755, 61.472, 62.2076, 63.1024, 63.8908, 64.7338, 65.7728, 66.629, 67.413, 68.3266, 69.1524, 70.2642, 71.1806, 72.0566, 72.9192, 73.7598, 74.3516, 75.5802, 76.4386, 77.4916, 78.1524, 79.1892, 79.8414, 80.8798, 81.8376, 82.4698, 83.7656, 84.331, 85.5914, 86.6012, 87.7016, 88.5582, 89.3394, 90.3544, 91.4912, 92.308, 93.3552, 93.9746, 95.2052, 95.727, 97.1322, 98.3944, 98.7588, 100.242, 101.1914, 102.2538, 102.8776, 103.6292, 105.1932, 105.9152, 107.0868, 107.6728, 108.7144, 110.3114, 110.8716, 111.245, 112.7908, 113.7064, 114.636, 115.7464, 116.1788, 117.7464, 118.4896, 119.6166, 120.5082, 121.7798, 122.9028, 123.4426, 124.8854, 125.705, 126.4652, 128.3464, 128.3462, 130.0398, 131.0342, 131.0042, 132.4766, 133.511, 134.7252, 135.425, 136.5172, 138.0572, 138.6694, 139.3712, 140.8598, 141.4594, 142.554, 143.4006, 144.7374, 146.1634, 146.8994, 147.605, 147.9304, 149.1636, 150.2468, 151.5876, 152.2096, 153.7032, 154.7146, 155.807, 156.9228, 157.0372, 158.5852, }, + // precision 6 + (double[]) { 46, 46.1902, 47.271, 47.8358, 48.8142, 49.2854, 50.317, 51.354, 51.8924, 52.9436, 53.4596, 54.5262, 55.6248, 56.1574, 57.2822, 57.837, 58.9636, 60.074, 60.7042, 61.7976, 62.4772, 63.6564, 64.7942, 65.5004, 66.686, 67.291, 68.5672, 69.8556, 70.4982, 71.8204, 72.4252, 73.7744, 75.0786, 75.8344, 77.0294, 77.8098, 79.0794, 80.5732, 81.1878, 82.5648, 83.2902, 84.6784, 85.3352, 86.8946, 88.3712, 89.0852, 90.499, 91.2686, 92.6844, 94.2234, 94.9732, 96.3356, 97.2286, 98.7262, 100.3284, 101.1048, 102.5962, 103.3562, 105.1272, 106.4184, 107.4974, 109.0822, 109.856, 111.48, 113.2834, 114.0208, 115.637, 116.5174, 118.0576, 119.7476, 120.427, 122.1326, 123.2372, 125.2788, 126.6776, 127.7926, 129.1952, 129.9564, 131.6454, 133.87, 134.5428, 136.2, 137.0294, 138.6278, 139.6782, 141.792, 143.3516, 144.2832, 146.0394, 147.0748, 148.4912, 150.849, 151.696, 153.5404, 154.073, 156.3714, 157.7216, 158.7328, 160.4208, 161.4184, 163.9424, 165.2772, 166.411, 168.1308, 168.769, 170.9258, 172.6828, 173.7502, 175.706, 176.3886, 179.0186, 180.4518, 181.927, 183.4172, 184.4114, 186.033, 188.5124, 189.5564, 191.6008, 192.4172, 193.8044, 194.997, 197.4548, 198.8948, 200.2346, 202.3086, 203.1548, 204.8842, 206.6508, 206.6772, 209.7254, 210.4752, 212.7228, 214.6614, 215.1676, 217.793, 218.0006, 219.9052, 221.66, 223.5588, 225.1636, 225.6882, 227.7126, 229.4502, 231.1978, 232.9756, 233.1654, 236.727, 238.1974, 237.7474, 241.1346, 242.3048, 244.1948, 245.3134, 246.879, 249.1204, 249.853, 252.6792, 253.857, 254.4486, 257.2362, 257.9534, 260.0286, 260.5632, 262.663, 264.723, 265.7566, 267.2566, 267.1624, 270.62, 272.8216, 273.2166, 275.2056, 276.2202, 278.3726, 280.3344, 281.9284, 283.9728, 284.1924, 286.4872, 287.587, 289.807, 291.1206, 292.769, 294.8708, 296.665, 297.1182, 299.4012, 300.6352, 302.1354, 304.1756, 306.1606, 307.3462, 308.5214, 309.4134, 310.8352, 313.9684, 315.837, 316.7796, 318.9858, }, + // precision 7 + (double[]) { 92, 93.4934, 94.9758, 96.4574, 97.9718, 99.4954, 101.5302, 103.0756, 104.6374, 106.1782, 107.7888, 109.9522, 111.592, 113.2532, 114.9086, 116.5938, 118.9474, 120.6796, 122.4394, 124.2176, 125.9768, 128.4214, 130.2528, 132.0102, 133.8658, 135.7278, 138.3044, 140.1316, 142.093, 144.0032, 145.9092, 148.6306, 150.5294, 152.5756, 154.6508, 156.662, 159.552, 161.3724, 163.617, 165.5754, 167.7872, 169.8444, 172.7988, 174.8606, 177.2118, 179.3566, 181.4476, 184.5882, 186.6816, 189.0824, 191.0258, 193.6048, 196.4436, 198.7274, 200.957, 203.147, 205.4364, 208.7592, 211.3386, 213.781, 215.8028, 218.656, 221.6544, 223.996, 226.4718, 229.1544, 231.6098, 234.5956, 237.0616, 239.5758, 242.4878, 244.5244, 248.2146, 250.724, 252.8722, 255.5198, 258.0414, 261.941, 264.9048, 266.87, 269.4304, 272.028, 274.4708, 278.37, 281.0624, 283.4668, 286.5532, 289.4352, 293.2564, 295.2744, 298.2118, 300.7472, 304.1456, 307.2928, 309.7504, 312.5528, 315.979, 318.2102, 322.1834, 324.3494, 327.325, 330.6614, 332.903, 337.2544, 339.9042, 343.215, 345.2864, 348.0814, 352.6764, 355.301, 357.139, 360.658, 363.1732, 366.5902, 369.9538, 373.0828, 375.922, 378.9902, 382.7328, 386.4538, 388.1136, 391.2234, 394.0878, 396.708, 401.1556, 404.1852, 406.6372, 409.6822, 412.7796, 416.6078, 418.4916, 422.131, 424.5376, 428.1988, 432.211, 434.4502, 438.5282, 440.912, 444.0448, 447.7432, 450.8524, 453.7988, 456.7858, 458.8868, 463.9886, 466.5064, 468.9124, 472.6616, 475.4682, 478.582, 481.304, 485.2738, 488.6894, 490.329, 496.106, 497.6908, 501.1374, 504.5322, 506.8848, 510.3324, 513.4512, 516.179, 520.4412, 522.6066, 526.167, 528.7794, 533.379, 536.067, 538.46, 542.9116, 545.692, 547.9546, 552.493, 555.2722, 557.335, 562.449, 564.2014, 569.0738, 571.0974, 574.8564, 578.2996, 581.409, 583.9704, 585.8098, 589.6528, 594.5998, 595.958, 600.068, 603.3278, 608.2016, 609.9632, 612.864, 615.43, 620.7794, 621.272, 625.8644, 629.206, 633.219, 634.5154, 638.6102, }, + // precision 8 + (double[]) { 184.2152, 187.2454, 190.2096, 193.6652, 196.6312, 199.6822, 203.249, 206.3296, 210.0038, 213.2074, 216.4612, 220.27, 223.5178, 227.4412, 230.8032, 234.1634, 238.1688, 241.6074, 245.6946, 249.2664, 252.8228, 257.0432, 260.6824, 264.9464, 268.6268, 272.2626, 276.8376, 280.4034, 284.8956, 288.8522, 292.7638, 297.3552, 301.3556, 305.7526, 309.9292, 313.8954, 318.8198, 322.7668, 327.298, 331.6688, 335.9466, 340.9746, 345.1672, 349.3474, 354.3028, 358.8912, 364.114, 368.4646, 372.9744, 378.4092, 382.6022, 387.843, 392.5684, 397.1652, 402.5426, 407.4152, 412.5388, 417.3592, 422.1366, 427.486, 432.3918, 437.5076, 442.509, 447.3834, 453.3498, 458.0668, 463.7346, 469.1228, 473.4528, 479.7, 484.644, 491.0518, 495.5774, 500.9068, 506.432, 512.1666, 517.434, 522.6644, 527.4894, 533.6312, 538.3804, 544.292, 550.5496, 556.0234, 562.8206, 566.6146, 572.4188, 579.117, 583.6762, 590.6576, 595.7864, 601.509, 607.5334, 612.9204, 619.772, 624.2924, 630.8654, 636.1836, 642.745, 649.1316, 655.0386, 660.0136, 666.6342, 671.6196, 678.1866, 684.4282, 689.3324, 695.4794, 702.5038, 708.129, 713.528, 720.3204, 726.463, 732.7928, 739.123, 744.7418, 751.2192, 756.5102, 762.6066, 769.0184, 775.2224, 781.4014, 787.7618, 794.1436, 798.6506, 805.6378, 811.766, 819.7514, 824.5776, 828.7322, 837.8048, 843.6302, 849.9336, 854.4798, 861.3388, 867.9894, 873.8196, 880.3136, 886.2308, 892.4588, 899.0816, 905.4076, 912.0064, 917.3878, 923.619, 929.998, 937.3482, 943.9506, 947.991, 955.1144, 962.203, 968.8222, 975.7324, 981.7826, 988.7666, 994.2648, 1000.3128, 1007.4082, 1013.7536, 1020.3376, 1026.7156, 1031.7478, 1037.4292, 1045.393, 1051.2278, 1058.3434, 1062.8726, 1071.884, 1076.806, 1082.9176, 1089.1678, 1095.5032, 1102.525, 1107.2264, 1115.315, 1120.93, 1127.252, 1134.1496, 1139.0408, 1147.5448, 1153.3296, 1158.1974, 1166.5262, 1174.3328, 1175.657, 1184.4222, 1190.9172, 1197.1292, 1204.4606, 1210.4578, 1218.8728, 1225.3336, 1226.6592, 1236.5768, 1241.363, 1249.4074, 1254.6566, 1260.8014, 1266.5454, 1274.5192, }, + // precision 9 + (double[]) { 369, 374.8294, 381.2452, 387.6698, 394.1464, 400.2024, 406.8782, 413.6598, 420.462, 427.2826, 433.7102, 440.7416, 447.9366, 455.1046, 462.285, 469.0668, 476.306, 483.8448, 491.301, 498.9886, 506.2422, 513.8138, 521.7074, 529.7428, 537.8402, 545.1664, 553.3534, 561.594, 569.6886, 577.7876, 585.65, 594.228, 602.8036, 611.1666, 620.0818, 628.0824, 637.2574, 646.302, 655.1644, 664.0056, 672.3802, 681.7192, 690.5234, 700.2084, 708.831, 718.485, 728.1112, 737.4764, 746.76, 756.3368, 766.5538, 775.5058, 785.2646, 795.5902, 804.3818, 814.8998, 824.9532, 835.2062, 845.2798, 854.4728, 864.9582, 875.3292, 886.171, 896.781, 906.5716, 916.7048, 927.5322, 937.875, 949.3972, 958.3464, 969.7274, 980.2834, 992.1444, 1003.4264, 1013.0166, 1024.018, 1035.0438, 1046.34, 1057.6856, 1068.9836, 1079.0312, 1091.677, 1102.3188, 1113.4846, 1124.4424, 1135.739, 1147.1488, 1158.9202, 1169.406, 1181.5342, 1193.2834, 1203.8954, 1216.3286, 1226.2146, 1239.6684, 1251.9946, 1262.123, 1275.4338, 1285.7378, 1296.076, 1308.9692, 1320.4964, 1333.0998, 1343.9864, 1357.7754, 1368.3208, 1380.4838, 1392.7388, 1406.0758, 1416.9098, 1428.9728, 1440.9228, 1453.9292, 1462.617, 1476.05, 1490.2996, 1500.6128, 1513.7392, 1524.5174, 1536.6322, 1548.2584, 1562.3766, 1572.423, 1587.1232, 1596.5164, 1610.5938, 1622.5972, 1633.1222, 1647.7674, 1658.5044, 1671.57, 1683.7044, 1695.4142, 1708.7102, 1720.6094, 1732.6522, 1747.841, 1756.4072, 1769.9786, 1782.3276, 1797.5216, 1808.3186, 1819.0694, 1834.354, 1844.575, 1856.2808, 1871.1288, 1880.7852, 1893.9622, 1906.3418, 1920.6548, 1932.9302, 1945.8584, 1955.473, 1968.8248, 1980.6446, 1995.9598, 2008.349, 2019.8556, 2033.0334, 2044.0206, 2059.3956, 2069.9174, 2082.6084, 2093.7036, 2106.6108, 2118.9124, 2132.301, 2144.7628, 2159.8422, 2171.0212, 2183.101, 2193.5112, 2208.052, 2221.3194, 2233.3282, 2247.295, 2257.7222, 2273.342, 2286.5638, 2299.6786, 2310.8114, 2322.3312, 2335.516, 2349.874, 2363.5968, 2373.865, 2387.1918, 2401.8328, 2414.8496, 2424.544, 2436.7592, 2447.1682, 2464.1958, 2474.3438, 2489.0006, 2497.4526, 2513.6586, 2527.19, 2540.7028, 2553.768 }, + // precision 10 + (double[]) { 738.1256, 750.4234, 763.1064, 775.4732, 788.4636, 801.0644, 814.488, 827.9654, 841.0832, 854.7864, 868.1992, 882.2176, 896.5228, 910.1716, 924.7752, 938.899, 953.6126, 968.6492, 982.9474, 998.5214, 1013.1064, 1028.6364, 1044.2468, 1059.4588, 1075.3832, 1091.0584, 1106.8606, 1123.3868, 1139.5062, 1156.1862, 1172.463, 1189.339, 1206.1936, 1223.1292, 1240.1854, 1257.2908, 1275.3324, 1292.8518, 1310.5204, 1328.4854, 1345.9318, 1364.552, 1381.4658, 1400.4256, 1419.849, 1438.152, 1456.8956, 1474.8792, 1494.118, 1513.62, 1532.5132, 1551.9322, 1570.7726, 1590.6086, 1610.5332, 1630.5918, 1650.4294, 1669.7662, 1690.4106, 1710.7338, 1730.9012, 1750.4486, 1770.1556, 1791.6338, 1812.7312, 1833.6264, 1853.9526, 1874.8742, 1896.8326, 1918.1966, 1939.5594, 1961.07, 1983.037, 2003.1804, 2026.071, 2047.4884, 2070.0848, 2091.2944, 2114.333, 2135.9626, 2158.2902, 2181.0814, 2202.0334, 2224.4832, 2246.39, 2269.7202, 2292.1714, 2314.2358, 2338.9346, 2360.891, 2384.0264, 2408.3834, 2430.1544, 2454.8684, 2476.9896, 2501.4368, 2522.8702, 2548.0408, 2570.6738, 2593.5208, 2617.0158, 2640.2302, 2664.0962, 2687.4986, 2714.2588, 2735.3914, 2759.6244, 2781.8378, 2808.0072, 2830.6516, 2856.2454, 2877.2136, 2903.4546, 2926.785, 2951.2294, 2976.468, 3000.867, 3023.6508, 3049.91, 3073.5984, 3098.162, 3121.5564, 3146.2328, 3170.9484, 3195.5902, 3221.3346, 3242.7032, 3271.6112, 3296.5546, 3317.7376, 3345.072, 3369.9518, 3394.326, 3418.1818, 3444.6926, 3469.086, 3494.2754, 3517.8698, 3544.248, 3565.3768, 3588.7234, 3616.979, 3643.7504, 3668.6812, 3695.72, 3719.7392, 3742.6224, 3770.4456, 3795.6602, 3819.9058, 3844.002, 3869.517, 3895.6824, 3920.8622, 3947.1364, 3973.985, 3995.4772, 4021.62, 4046.628, 4074.65, 4096.2256, 4121.831, 4146.6406, 4173.276, 4195.0744, 4223.9696, 4251.3708, 4272.9966, 4300.8046, 4326.302, 4353.1248, 4374.312, 4403.0322, 4426.819, 4450.0598, 4478.5206, 4504.8116, 4528.8928, 4553.9584, 4578.8712, 4603.8384, 4632.3872, 4655.5128, 4675.821, 4704.6222, 4731.9862, 4755.4174, 4781.2628, 4804.332, 4832.3048, 4862.8752, 4883.4148, 4906.9544, 4935.3516, 4954.3532, 4984.0248, 5011.217, 5035.3258, 5057.3672, 5084.1828, }, + // precision 11 + (double[]) { 1477, 1501.6014, 1526.5802, 1551.7942, 1577.3042, 1603.2062, 1629.8402, 1656.2292, 1682.9462, 1709.9926, 1737.3026, 1765.4252, 1793.0578, 1821.6092, 1849.626, 1878.5568, 1908.527, 1937.5154, 1967.1874, 1997.3878, 2027.37, 2058.1972, 2089.5728, 2120.1012, 2151.9668, 2183.292, 2216.0772, 2247.8578, 2280.6562, 2313.041, 2345.714, 2380.3112, 2414.1806, 2447.9854, 2481.656, 2516.346, 2551.5154, 2586.8378, 2621.7448, 2656.6722, 2693.5722, 2729.1462, 2765.4124, 2802.8728, 2838.898, 2876.408, 2913.4926, 2951.4938, 2989.6776, 3026.282, 3065.7704, 3104.1012, 3143.7388, 3181.6876, 3221.1872, 3261.5048, 3300.0214, 3339.806, 3381.409, 3421.4144, 3461.4294, 3502.2286, 3544.651, 3586.6156, 3627.337, 3670.083, 3711.1538, 3753.5094, 3797.01, 3838.6686, 3882.1678, 3922.8116, 3967.9978, 4009.9204, 4054.3286, 4097.5706, 4140.6014, 4185.544, 4229.5976, 4274.583, 4316.9438, 4361.672, 4406.2786, 4451.8628, 4496.1834, 4543.505, 4589.1816, 4632.5188, 4678.2294, 4724.8908, 4769.0194, 4817.052, 4861.4588, 4910.1596, 4956.4344, 5002.5238, 5048.13, 5093.6374, 5142.8162, 5187.7894, 5237.3984, 5285.6078, 5331.0858, 5379.1036, 5428.6258, 5474.6018, 5522.7618, 5571.5822, 5618.59, 5667.9992, 5714.88, 5763.454, 5808.6982, 5860.3644, 5910.2914, 5953.571, 6005.9232, 6055.1914, 6104.5882, 6154.5702, 6199.7036, 6251.1764, 6298.7596, 6350.0302, 6398.061, 6448.4694, 6495.933, 6548.0474, 6597.7166, 6646.9416, 6695.9208, 6742.6328, 6793.5276, 6842.1934, 6894.2372, 6945.3864, 6996.9228, 7044.2372, 7094.1374, 7142.2272, 7192.2942, 7238.8338, 7288.9006, 7344.0908, 7394.8544, 7443.5176, 7490.4148, 7542.9314, 7595.6738, 7641.9878, 7694.3688, 7743.0448, 7797.522, 7845.53, 7899.594, 7950.3132, 7996.455, 8050.9442, 8092.9114, 8153.1374, 8197.4472, 8252.8278, 8301.8728, 8348.6776, 8401.4698, 8453.551, 8504.6598, 8553.8944, 8604.1276, 8657.6514, 8710.3062, 8758.908, 8807.8706, 8862.1702, 8910.4668, 8960.77, 9007.2766, 9063.164, 9121.0534, 9164.1354, 9218.1594, 9267.767, 9319.0594, 9372.155, 9419.7126, 9474.3722, 9520.1338, 9572.368, 9622.7702, 9675.8448, 9726.5396, 9778.7378, 9827.6554, 9878.1922, 9928.7782, 9978.3984, 10026.578, 10076.5626, 10137.1618, 10177.5244, 10229.9176, }, + // precision 12 + (double[]) { 2954, 3003.4782, 3053.3568, 3104.3666, 3155.324, 3206.9598, 3259.648, 3312.539, 3366.1474, 3420.2576, 3474.8376, 3530.6076, 3586.451, 3643.38, 3700.4104, 3757.5638, 3815.9676, 3875.193, 3934.838, 3994.8548, 4055.018, 4117.1742, 4178.4482, 4241.1294, 4304.4776, 4367.4044, 4431.8724, 4496.3732, 4561.4304, 4627.5326, 4693.949, 4761.5532, 4828.7256, 4897.6182, 4965.5186, 5034.4528, 5104.865, 5174.7164, 5244.6828, 5316.6708, 5387.8312, 5459.9036, 5532.476, 5604.8652, 5679.6718, 5753.757, 5830.2072, 5905.2828, 5980.0434, 6056.6264, 6134.3192, 6211.5746, 6290.0816, 6367.1176, 6447.9796, 6526.5576, 6606.1858, 6686.9144, 6766.1142, 6847.0818, 6927.9664, 7010.9096, 7091.0816, 7175.3962, 7260.3454, 7344.018, 7426.4214, 7511.3106, 7596.0686, 7679.8094, 7765.818, 7852.4248, 7936.834, 8022.363, 8109.5066, 8200.4554, 8288.5832, 8373.366, 8463.4808, 8549.7682, 8642.0522, 8728.3288, 8820.9528, 8907.727, 9001.0794, 9091.2522, 9179.988, 9269.852, 9362.6394, 9453.642, 9546.9024, 9640.6616, 9732.6622, 9824.3254, 9917.7484, 10007.9392, 10106.7508, 10196.2152, 10289.8114, 10383.5494, 10482.3064, 10576.8734, 10668.7872, 10764.7156, 10862.0196, 10952.793, 11049.9748, 11146.0702, 11241.4492, 11339.2772, 11434.2336, 11530.741, 11627.6136, 11726.311, 11821.5964, 11918.837, 12015.3724, 12113.0162, 12213.0424, 12306.9804, 12408.4518, 12504.8968, 12604.586, 12700.9332, 12798.705, 12898.5142, 12997.0488, 13094.788, 13198.475, 13292.7764, 13392.9698, 13486.8574, 13590.1616, 13686.5838, 13783.6264, 13887.2638, 13992.0978, 14081.0844, 14189.9956, 14280.0912, 14382.4956, 14486.4384, 14588.1082, 14686.2392, 14782.276, 14888.0284, 14985.1864, 15088.8596, 15187.0998, 15285.027, 15383.6694, 15495.8266, 15591.3736, 15694.2008, 15790.3246, 15898.4116, 15997.4522, 16095.5014, 16198.8514, 16291.7492, 16402.6424, 16499.1266, 16606.2436, 16697.7186, 16796.3946, 16902.3376, 17005.7672, 17100.814, 17206.8282, 17305.8262, 17416.0744, 17508.4092, 17617.0178, 17715.4554, 17816.758, 17920.1748, 18012.9236, 18119.7984, 18223.2248, 18324.2482, 18426.6276, 18525.0932, 18629.8976, 18733.2588, 18831.0466, 18940.1366, 19032.2696, 19131.729, 19243.4864, 19349.6932, 19442.866, 19547.9448, 19653.2798, 19754.4034, 19854.0692, 19965.1224, 20065.1774, 20158.2212, 20253.353, 20366.3264, 20463.22, }, + // precision 13 + (double[]) { 5908.5052, 6007.2672, 6107.347, 6208.5794, 6311.2622, 6414.5514, 6519.3376, 6625.6952, 6732.5988, 6841.3552, 6950.5972, 7061.3082, 7173.5646, 7287.109, 7401.8216, 7516.4344, 7633.3802, 7751.2962, 7870.3784, 7990.292, 8110.79, 8233.4574, 8356.6036, 8482.2712, 8607.7708, 8735.099, 8863.1858, 8993.4746, 9123.8496, 9255.6794, 9388.5448, 9522.7516, 9657.3106, 9792.6094, 9930.5642, 10068.794, 10206.7256, 10347.81, 10490.3196, 10632.0778, 10775.9916, 10920.4662, 11066.124, 11213.073, 11358.0362, 11508.1006, 11659.1716, 11808.7514, 11959.4884, 12112.1314, 12265.037, 12420.3756, 12578.933, 12734.311, 12890.0006, 13047.2144, 13207.3096, 13368.5144, 13528.024, 13689.847, 13852.7528, 14018.3168, 14180.5372, 14346.9668, 14513.5074, 14677.867, 14846.2186, 15017.4186, 15184.9716, 15356.339, 15529.2972, 15697.3578, 15871.8686, 16042.187, 16216.4094, 16389.4188, 16565.9126, 16742.3272, 16919.0042, 17094.7592, 17273.965, 17451.8342, 17634.4254, 17810.5984, 17988.9242, 18171.051, 18354.7938, 18539.466, 18721.0408, 18904.9972, 19081.867, 19271.9118, 19451.8694, 19637.9816, 19821.2922, 20013.1292, 20199.3858, 20387.8726, 20572.9514, 20770.7764, 20955.1714, 21144.751, 21329.9952, 21520.709, 21712.7016, 21906.3868, 22096.2626, 22286.0524, 22475.051, 22665.5098, 22862.8492, 23055.5294, 23249.6138, 23437.848, 23636.273, 23826.093, 24020.3296, 24213.3896, 24411.7392, 24602.9614, 24805.7952, 24998.1552, 25193.9588, 25389.0166, 25585.8392, 25780.6976, 25981.2728, 26175.977, 26376.5252, 26570.1964, 26773.387, 26962.9812, 27163.0586, 27368.164, 27565.0534, 27758.7428, 27961.1276, 28163.2324, 28362.3816, 28565.7668, 28758.644, 28956.9768, 29163.4722, 29354.7026, 29561.1186, 29767.9948, 29959.9986, 30164.0492, 30366.9818, 30562.5338, 30762.9928, 30976.1592, 31166.274, 31376.722, 31570.3734, 31770.809, 31974.8934, 32179.5286, 32387.5442, 32582.3504, 32794.076, 32989.9528, 33191.842, 33392.4684, 33595.659, 33801.8672, 34000.3414, 34200.0922, 34402.6792, 34610.0638, 34804.0084, 35011.13, 35218.669, 35418.6634, 35619.0792, 35830.6534, 36028.4966, 36229.7902, 36438.6422, 36630.7764, 36833.3102, 37048.6728, 37247.3916, 37453.5904, 37669.3614, 37854.5526, 38059.305, 38268.0936, 38470.2516, 38674.7064, 38876.167, 39068.3794, 39281.9144, 39492.8566, 39684.8628, 39898.4108, 40093.1836, 40297.6858, 40489.7086, 40717.2424, }, + // precision 14 + (double[]) { 11817.475, 12015.0046, 12215.3792, 12417.7504, 12623.1814, 12830.0086, 13040.0072, 13252.503, 13466.178, 13683.2738, 13902.0344, 14123.9798, 14347.394, 14573.7784, 14802.6894, 15033.6824, 15266.9134, 15502.8624, 15741.4944, 15980.7956, 16223.8916, 16468.6316, 16715.733, 16965.5726, 17217.204, 17470.666, 17727.8516, 17986.7886, 18247.6902, 18510.9632, 18775.304, 19044.7486, 19314.4408, 19587.202, 19862.2576, 20135.924, 20417.0324, 20697.9788, 20979.6112, 21265.0274, 21550.723, 21841.6906, 22132.162, 22428.1406, 22722.127, 23020.5606, 23319.7394, 23620.4014, 23925.2728, 24226.9224, 24535.581, 24845.505, 25155.9618, 25470.3828, 25785.9702, 26103.7764, 26420.4132, 26742.0186, 27062.8852, 27388.415, 27714.6024, 28042.296, 28365.4494, 28701.1526, 29031.8008, 29364.2156, 29704.497, 30037.1458, 30380.111, 30723.8168, 31059.5114, 31404.9498, 31751.6752, 32095.2686, 32444.7792, 32794.767, 33145.204, 33498.4226, 33847.6502, 34209.006, 34560.849, 34919.4838, 35274.9778, 35635.1322, 35996.3266, 36359.1394, 36722.8266, 37082.8516, 37447.7354, 37815.9606, 38191.0692, 38559.4106, 38924.8112, 39294.6726, 39663.973, 40042.261, 40416.2036, 40779.2036, 41161.6436, 41540.9014, 41921.1998, 42294.7698, 42678.5264, 43061.3464, 43432.375, 43818.432, 44198.6598, 44583.0138, 44970.4794, 45353.924, 45729.858, 46118.2224, 46511.5724, 46900.7386, 47280.6964, 47668.1472, 48055.6796, 48446.9436, 48838.7146, 49217.7296, 49613.7796, 50010.7508, 50410.0208, 50793.7886, 51190.2456, 51583.1882, 51971.0796, 52376.5338, 52763.319, 53165.5534, 53556.5594, 53948.2702, 54346.352, 54748.7914, 55138.577, 55543.4824, 55941.1748, 56333.7746, 56745.1552, 57142.7944, 57545.2236, 57935.9956, 58348.5268, 58737.5474, 59158.5962, 59542.6896, 59958.8004, 60349.3788, 60755.0212, 61147.6144, 61548.194, 61946.0696, 62348.6042, 62763.603, 63162.781, 63560.635, 63974.3482, 64366.4908, 64771.5876, 65176.7346, 65597.3916, 65995.915, 66394.0384, 66822.9396, 67203.6336, 67612.2032, 68019.0078, 68420.0388, 68821.22, 69235.8388, 69640.0724, 70055.155, 70466.357, 70863.4266, 71276.2482, 71677.0306, 72080.2006, 72493.0214, 72893.5952, 73314.5856, 73714.9852, 74125.3022, 74521.2122, 74933.6814, 75341.5904, 75743.0244, 76166.0278, 76572.1322, 76973.1028, 77381.6284, 77800.6092, 78189.328, 78607.0962, 79012.2508, 79407.8358, 79825.725, 80238.701, 80646.891, 81035.6436, 81460.0448, 81876.3884, }, + // precision 15 + (double[]) { 23635.0036, 24030.8034, 24431.4744, 24837.1524, 25246.7928, 25661.326, 26081.3532, 26505.2806, 26933.9892, 27367.7098, 27805.318, 28248.799, 28696.4382, 29148.8244, 29605.5138, 30066.8668, 30534.2344, 31006.32, 31480.778, 31962.2418, 32447.3324, 32938.0232, 33432.731, 33930.728, 34433.9896, 34944.1402, 35457.5588, 35974.5958, 36497.3296, 37021.9096, 37554.326, 38088.0826, 38628.8816, 39171.3192, 39723.2326, 40274.5554, 40832.3142, 41390.613, 41959.5908, 42532.5466, 43102.0344, 43683.5072, 44266.694, 44851.2822, 45440.7862, 46038.0586, 46640.3164, 47241.064, 47846.155, 48454.7396, 49076.9168, 49692.542, 50317.4778, 50939.65, 51572.5596, 52210.2906, 52843.7396, 53481.3996, 54127.236, 54770.406, 55422.6598, 56078.7958, 56736.7174, 57397.6784, 58064.5784, 58730.308, 59404.9784, 60077.0864, 60751.9158, 61444.1386, 62115.817, 62808.7742, 63501.4774, 64187.5454, 64883.6622, 65582.7468, 66274.5318, 66976.9276, 67688.7764, 68402.138, 69109.6274, 69822.9706, 70543.6108, 71265.5202, 71983.3848, 72708.4656, 73433.384, 74158.4664, 74896.4868, 75620.9564, 76362.1434, 77098.3204, 77835.7662, 78582.6114, 79323.9902, 80067.8658, 80814.9246, 81567.0136, 82310.8536, 83061.9952, 83821.4096, 84580.8608, 85335.547, 86092.5802, 86851.6506, 87612.311, 88381.2016, 89146.3296, 89907.8974, 90676.846, 91451.4152, 92224.5518, 92995.8686, 93763.5066, 94551.2796, 95315.1944, 96096.1806, 96881.0918, 97665.679, 98442.68, 99229.3002, 100011.0994, 100790.6386, 101580.1564, 102377.7484, 103152.1392, 103944.2712, 104730.216, 105528.6336, 106324.9398, 107117.6706, 107890.3988, 108695.2266, 109485.238, 110294.7876, 111075.0958, 111878.0496, 112695.2864, 113464.5486, 114270.0474, 115068.608, 115884.3626, 116673.2588, 117483.3716, 118275.097, 119085.4092, 119879.2808, 120687.5868, 121499.9944, 122284.916, 123095.9254, 123912.5038, 124709.0454, 125503.7182, 126323.259, 127138.9412, 127943.8294, 128755.646, 129556.5354, 130375.3298, 131161.4734, 131971.1962, 132787.5458, 133588.1056, 134431.351, 135220.2906, 136023.398, 136846.6558, 137667.0004, 138463.663, 139283.7154, 140074.6146, 140901.3072, 141721.8548, 142543.2322, 143356.1096, 144173.7412, 144973.0948, 145794.3162, 146609.5714, 147420.003, 148237.9784, 149050.5696, 149854.761, 150663.1966, 151494.0754, 152313.1416, 153112.6902, 153935.7206, 154746.9262, 155559.547, 156401.9746, 157228.7036, 158008.7254, 158820.75, 159646.9184, 160470.4458, 161279.5348, 162093.3114, 162918.542, 163729.2842, }, + // precision 16 + (double[]) { 47271, 48062.3584, 48862.7074, 49673.152, 50492.8416, 51322.9514, 52161.03, 53009.407, 53867.6348, 54734.206, 55610.5144, 56496.2096, 57390.795, 58297.268, 59210.6448, 60134.665, 61068.0248, 62010.4472, 62962.5204, 63923.5742, 64895.0194, 65876.4182, 66862.6136, 67862.6968, 68868.8908, 69882.8544, 70911.271, 71944.0924, 72990.0326, 74040.692, 75100.6336, 76174.7826, 77252.5998, 78340.2974, 79438.2572, 80545.4976, 81657.2796, 82784.6336, 83915.515, 85059.7362, 86205.9368, 87364.4424, 88530.3358, 89707.3744, 90885.9638, 92080.197, 93275.5738, 94479.391, 95695.918, 96919.2236, 98148.4602, 99382.3474, 100625.6974, 101878.0284, 103141.6278, 104409.4588, 105686.2882, 106967.5402, 108261.6032, 109548.1578, 110852.0728, 112162.231, 113479.0072, 114806.2626, 116137.9072, 117469.5048, 118813.5186, 120165.4876, 121516.2556, 122875.766, 124250.5444, 125621.2222, 127003.2352, 128387.848, 129775.2644, 131181.7776, 132577.3086, 133979.9458, 135394.1132, 136800.9078, 138233.217, 139668.5308, 141085.212, 142535.2122, 143969.0684, 145420.2872, 146878.1542, 148332.7572, 149800.3202, 151269.66, 152743.6104, 154213.0948, 155690.288, 157169.4246, 158672.1756, 160160.059, 161650.6854, 163145.7772, 164645.6726, 166159.1952, 167682.1578, 169177.3328, 170700.0118, 172228.8964, 173732.6664, 175265.5556, 176787.799, 178317.111, 179856.6914, 181400.865, 182943.4612, 184486.742, 186033.4698, 187583.7886, 189148.1868, 190688.4526, 192250.1926, 193810.9042, 195354.2972, 196938.7682, 198493.5898, 200079.2824, 201618.912, 203205.5492, 204765.5798, 206356.1124, 207929.3064, 209498.7196, 211086.229, 212675.1324, 214256.7892, 215826.2392, 217412.8474, 218995.6724, 220618.6038, 222207.1166, 223781.0364, 225387.4332, 227005.7928, 228590.4336, 230217.8738, 231805.1054, 233408.9, 234995.3432, 236601.4956, 238190.7904, 239817.2548, 241411.2832, 243002.4066, 244640.1884, 246255.3128, 247849.3508, 249479.9734, 251106.8822, 252705.027, 254332.9242, 255935.129, 257526.9014, 259154.772, 260777.625, 262390.253, 264004.4906, 265643.59, 267255.4076, 268873.426, 270470.7252, 272106.4804, 273722.4456, 275337.794, 276945.7038, 278592.9154, 280204.3726, 281841.1606, 283489.171, 285130.1716, 286735.3362, 288364.7164, 289961.1814, 291595.5524, 293285.683, 294899.6668, 296499.3434, 298128.0462, 299761.8946, 301394.2424, 302997.6748, 304615.1478, 306269.7724, 307886.114, 309543.1028, 311153.2862, 312782.8546, 314421.2008, 316033.2438, 317692.9636, 319305.2648, 320948.7406, 322566.3364, 324228.4224, 325847.1542, }, + // precision 17 + (double[]) { 94542, 96125.811, 97728.019, 99348.558, 100987.9705, 102646.7565, 104324.5125, 106021.7435, 107736.7865, 109469.272, 111223.9465, 112995.219, 114787.432, 116593.152, 118422.71, 120267.2345, 122134.6765, 124020.937, 125927.2705, 127851.255, 129788.9485, 131751.016, 133726.8225, 135722.592, 137736.789, 139770.568, 141821.518, 143891.343, 145982.1415, 148095.387, 150207.526, 152355.649, 154515.6415, 156696.05, 158887.7575, 161098.159, 163329.852, 165569.053, 167837.4005, 170121.6165, 172420.4595, 174732.6265, 177062.77, 179412.502, 181774.035, 184151.939, 186551.6895, 188965.691, 191402.8095, 193857.949, 196305.0775, 198774.6715, 201271.2585, 203764.78, 206299.3695, 208818.1365, 211373.115, 213946.7465, 216532.076, 219105.541, 221714.5375, 224337.5135, 226977.5125, 229613.0655, 232270.2685, 234952.2065, 237645.3555, 240331.1925, 243034.517, 245756.0725, 248517.6865, 251232.737, 254011.3955, 256785.995, 259556.44, 262368.335, 265156.911, 267965.266, 270785.583, 273616.0495, 276487.4835, 279346.639, 282202.509, 285074.3885, 287942.2855, 290856.018, 293774.0345, 296678.5145, 299603.6355, 302552.6575, 305492.9785, 308466.8605, 311392.581, 314347.538, 317319.4295, 320285.9785, 323301.7325, 326298.3235, 329301.3105, 332301.987, 335309.791, 338370.762, 341382.923, 344431.1265, 347464.1545, 350507.28, 353619.2345, 356631.2005, 359685.203, 362776.7845, 365886.488, 368958.2255, 372060.6825, 375165.4335, 378237.935, 381328.311, 384430.5225, 387576.425, 390683.242, 393839.648, 396977.8425, 400101.9805, 403271.296, 406409.8425, 409529.5485, 412678.7, 415847.423, 419020.8035, 422157.081, 425337.749, 428479.6165, 431700.902, 434893.1915, 438049.582, 441210.5415, 444379.2545, 447577.356, 450741.931, 453959.548, 457137.0935, 460329.846, 463537.4815, 466732.3345, 469960.5615, 473164.681, 476347.6345, 479496.173, 482813.1645, 486025.6995, 489249.4885, 492460.1945, 495675.8805, 498908.0075, 502131.802, 505374.3855, 508550.9915, 511806.7305, 515026.776, 518217.0005, 521523.9855, 524705.9855, 527950.997, 531210.0265, 534472.497, 537750.7315, 540926.922, 544207.094, 547429.4345, 550666.3745, 553975.3475, 557150.7185, 560399.6165, 563662.697, 566916.7395, 570146.1215, 573447.425, 576689.6245, 579874.5745, 583202.337, 586503.0255, 589715.635, 592910.161, 596214.3885, 599488.035, 602740.92, 605983.0685, 609248.67, 612491.3605, 615787.912, 619107.5245, 622307.9555, 625577.333, 628840.4385, 632085.2155, 635317.6135, 638691.7195, 641887.467, 645139.9405, 648441.546, 651666.252, 654941.845, }, + // precision 18 + (double[]) { 189084, 192250.913, 195456.774, 198696.946, 201977.762, 205294.444, 208651.754, 212042.099, 215472.269, 218941.91, 222443.912, 225996.845, 229568.199, 233193.568, 236844.457, 240543.233, 244279.475, 248044.27, 251854.588, 255693.2, 259583.619, 263494.621, 267445.385, 271454.061, 275468.769, 279549.456, 283646.446, 287788.198, 291966.099, 296181.164, 300431.469, 304718.618, 309024.004, 313393.508, 317760.803, 322209.731, 326675.061, 331160.627, 335654.47, 340241.442, 344841.833, 349467.132, 354130.629, 358819.432, 363574.626, 368296.587, 373118.482, 377914.93, 382782.301, 387680.669, 392601.981, 397544.323, 402529.115, 407546.018, 412593.658, 417638.657, 422762.865, 427886.169, 433017.167, 438213.273, 443441.254, 448692.421, 453937.533, 459239.049, 464529.569, 469910.083, 475274.03, 480684.473, 486070.26, 491515.237, 496995.651, 502476.617, 507973.609, 513497.19, 519083.233, 524726.509, 530305.505, 535945.728, 541584.404, 547274.055, 552967.236, 558667.862, 564360.216, 570128.148, 575965.08, 581701.952, 587532.523, 593361.144, 599246.128, 605033.418, 610958.779, 616837.117, 622772.818, 628672.04, 634675.369, 640574.831, 646585.739, 652574.547, 658611.217, 664642.684, 670713.914, 676737.681, 682797.313, 688837.897, 694917.874, 701009.882, 707173.648, 713257.254, 719415.392, 725636.761, 731710.697, 737906.209, 744103.074, 750313.39, 756504.185, 762712.579, 768876.985, 775167.859, 781359, 787615.959, 793863.597, 800245.477, 806464.582, 812785.294, 819005.925, 825403.057, 831676.197, 837936.284, 844266.968, 850642.711, 856959.756, 863322.774, 869699.931, 876102.478, 882355.787, 888694.463, 895159.952, 901536.143, 907872.631, 914293.672, 920615.14, 927130.974, 933409.404, 939922.178, 946331.47, 952745.93, 959209.264, 965590.224, 972077.284, 978501.961, 984953.19, 991413.271, 997817.479, 1004222.658, 1010725.676, 1017177.138, 1023612.529, 1030098.236, 1036493.719, 1043112.207, 1049537.036, 1056008.096, 1062476.184, 1068942.337, 1075524.95, 1081932.864, 1088426.025, 1094776.005, 1101327.448, 1107901.673, 1114423.639, 1120884.602, 1127324.923, 1133794.24, 1140328.886, 1146849.376, 1153346.682, 1159836.502, 1166478.703, 1172953.304, 1179391.502, 1185950.982, 1192544.052, 1198913.41, 1205430.994, 1212015.525, 1218674.042, 1225121.683, 1231551.101, 1238126.379, 1244673.795, 1251260.649, 1257697.86, 1264320.983, 1270736.319, 1277274.694, 1283804.95, 1290211.514, 1296858.568, 1303455.691, } +}; + +static const double *g_biasData[] = { + // precision 4 + (double[]) { 10, 9.717, 9.207, 8.7896, 8.2882, 7.8204, 7.3772, 6.9342, 6.5202, 6.161, 5.7722, 5.4636, 5.0396, 4.6766, 4.3566, 4.0454, 3.7936, 3.4856, 3.2666, 2.9946, 2.766, 2.4692, 2.3638, 2.0764, 1.7864, 1.7602, 1.4814, 1.433, 1.2926, 1.0664, 0.999600000000001, 0.7956, 0.5366, 0.589399999999998, 0.573799999999999, 0.269799999999996, 0.368200000000002, 0.0544000000000011, 0.234200000000001, 0.0108000000000033, -0.203400000000002, -0.0701999999999998, -0.129600000000003, -0.364199999999997, -0.480600000000003, -0.226999999999997, -0.322800000000001, -0.382599999999996, -0.511200000000002, -0.669600000000003, -0.749400000000001, -0.500399999999999, -0.617600000000003, -0.6922, -0.601599999999998, -0.416200000000003, -0.338200000000001, -0.782600000000002, -0.648600000000002, -0.919800000000002, -0.851799999999997, -0.962400000000002, -0.6402, -1.1922, -1.0256, -1.086, -1.21899999999999, -0.819400000000002, -0.940600000000003, -1.1554, -1.2072, -1.1752, -1.16759999999999, -1.14019999999999, -1.3754, -1.29859999999999, -1.607, -1.3292, -1.7606, }, + // precision 5 + (double[]) { 22, 21.1194, 20.8208, 20.2318, 19.77, 19.2436, 18.7774, 18.2848, 17.8224, 17.3742, 16.9336, 16.503, 16.0494, 15.6292, 15.2124, 14.798, 14.367, 13.9728, 13.5944, 13.217, 12.8438, 12.3696, 12.0956, 11.7044, 11.324, 11.0668, 10.6698, 10.3644, 10.049, 9.6918, 9.4146, 9.082, 8.687, 8.5398, 8.2462, 7.857, 7.6606, 7.4168, 7.1248, 6.9222, 6.6804, 6.447, 6.3454, 5.9594, 5.7636, 5.5776, 5.331, 5.19, 4.9676, 4.7564, 4.5314, 4.4442, 4.3708, 3.9774, 3.9624, 3.8796, 3.755, 3.472, 3.2076, 3.1024, 2.8908, 2.7338, 2.7728, 2.629, 2.413, 2.3266, 2.1524, 2.2642, 2.1806, 2.0566, 1.9192, 1.7598, 1.3516, 1.5802, 1.43859999999999, 1.49160000000001, 1.1524, 1.1892, 0.841399999999993, 0.879800000000003, 0.837599999999995, 0.469800000000006, 0.765600000000006, 0.331000000000003, 0.591399999999993, 0.601200000000006, 0.701599999999999, 0.558199999999999, 0.339399999999998, 0.354399999999998, 0.491200000000006, 0.308000000000007, 0.355199999999996, -0.0254000000000048, 0.205200000000005, -0.272999999999996, 0.132199999999997, 0.394400000000005, -0.241200000000006, 0.242000000000004, 0.191400000000002, 0.253799999999998, -0.122399999999999, -0.370800000000003, 0.193200000000004, -0.0848000000000013, 0.0867999999999967, -0.327200000000005, -0.285600000000002, 0.311400000000006, -0.128399999999999, -0.754999999999995, -0.209199999999996, -0.293599999999998, -0.364000000000004, -0.253600000000006, -0.821200000000005, -0.253600000000006, -0.510400000000004, -0.383399999999995, -0.491799999999998, -0.220200000000006, -0.0972000000000008, -0.557400000000001, -0.114599999999996, -0.295000000000002, -0.534800000000004, 0.346399999999988, -0.65379999999999, 0.0398000000000138, 0.0341999999999985, -0.995800000000003, -0.523400000000009, -0.489000000000004, -0.274799999999999, -0.574999999999989, -0.482799999999997, 0.0571999999999946, -0.330600000000004, -0.628800000000012, -0.140199999999993, -0.540600000000012, -0.445999999999998, -0.599400000000003, -0.262599999999992, 0.163399999999996, -0.100599999999986, -0.39500000000001, -1.06960000000001, -0.836399999999998, -0.753199999999993, -0.412399999999991, -0.790400000000005, -0.29679999999999, -0.28540000000001, -0.193000000000012, -0.0772000000000048, -0.962799999999987, -0.414800000000014, }, + // precision 6 + (double[]) { 45, 44.1902, 43.271, 42.8358, 41.8142, 41.2854, 40.317, 39.354, 38.8924, 37.9436, 37.4596, 36.5262, 35.6248, 35.1574, 34.2822, 33.837, 32.9636, 32.074, 31.7042, 30.7976, 30.4772, 29.6564, 28.7942, 28.5004, 27.686, 27.291, 26.5672, 25.8556, 25.4982, 24.8204, 24.4252, 23.7744, 23.0786, 22.8344, 22.0294, 21.8098, 21.0794, 20.5732, 20.1878, 19.5648, 19.2902, 18.6784, 18.3352, 17.8946, 17.3712, 17.0852, 16.499, 16.2686, 15.6844, 15.2234, 14.9732, 14.3356, 14.2286, 13.7262, 13.3284, 13.1048, 12.5962, 12.3562, 12.1272, 11.4184, 11.4974, 11.0822, 10.856, 10.48, 10.2834, 10.0208, 9.637, 9.51739999999999, 9.05759999999999, 8.74760000000001, 8.42700000000001, 8.1326, 8.2372, 8.2788, 7.6776, 7.79259999999999, 7.1952, 6.9564, 6.6454, 6.87, 6.5428, 6.19999999999999, 6.02940000000001, 5.62780000000001, 5.6782, 5.792, 5.35159999999999, 5.28319999999999, 5.0394, 5.07480000000001, 4.49119999999999, 4.84899999999999, 4.696, 4.54040000000001, 4.07300000000001, 4.37139999999999, 3.7216, 3.7328, 3.42080000000001, 3.41839999999999, 3.94239999999999, 3.27719999999999, 3.411, 3.13079999999999, 2.76900000000001, 2.92580000000001, 2.68279999999999, 2.75020000000001, 2.70599999999999, 2.3886, 3.01859999999999, 2.45179999999999, 2.92699999999999, 2.41720000000001, 2.41139999999999, 2.03299999999999, 2.51240000000001, 2.5564, 2.60079999999999, 2.41720000000001, 1.80439999999999, 1.99700000000001, 2.45480000000001, 1.8948, 2.2346, 2.30860000000001, 2.15479999999999, 1.88419999999999, 1.6508, 0.677199999999999, 1.72540000000001, 1.4752, 1.72280000000001, 1.66139999999999, 1.16759999999999, 1.79300000000001, 1.00059999999999, 0.905200000000008, 0.659999999999997, 1.55879999999999, 1.1636, 0.688199999999995, 0.712600000000009, 0.450199999999995, 1.1978, 0.975599999999986, 0.165400000000005, 1.727, 1.19739999999999, -0.252600000000001, 1.13460000000001, 1.3048, 1.19479999999999, 0.313400000000001, 0.878999999999991, 1.12039999999999, 0.853000000000009, 1.67920000000001, 0.856999999999999, 0.448599999999999, 1.2362, 0.953399999999988, 1.02859999999998, 0.563199999999995, 0.663000000000011, 0.723000000000013, 0.756599999999992, 0.256599999999992, -0.837600000000009, 0.620000000000005, 0.821599999999989, 0.216600000000028, 0.205600000000004, 0.220199999999977, 0.372599999999977, 0.334400000000016, 0.928400000000011, 0.972800000000007, 0.192400000000021, 0.487199999999973, -0.413000000000011, 0.807000000000016, 0.120600000000024, 0.769000000000005, 0.870799999999974, 0.66500000000002, 0.118200000000002, 0.401200000000017, 0.635199999999998, 0.135400000000004, 0.175599999999974, 1.16059999999999, 0.34620000000001, 0.521400000000028, -0.586599999999976, -1.16480000000001, 0.968399999999974, 0.836999999999989, 0.779600000000016, 0.985799999999983, }, + // precision 7 + (double[]) { 91, 89.4934, 87.9758, 86.4574, 84.9718, 83.4954, 81.5302, 80.0756, 78.6374, 77.1782, 75.7888, 73.9522, 72.592, 71.2532, 69.9086, 68.5938, 66.9474, 65.6796, 64.4394, 63.2176, 61.9768, 60.4214, 59.2528, 58.0102, 56.8658, 55.7278, 54.3044, 53.1316, 52.093, 51.0032, 49.9092, 48.6306, 47.5294, 46.5756, 45.6508, 44.662, 43.552, 42.3724, 41.617, 40.5754, 39.7872, 38.8444, 37.7988, 36.8606, 36.2118, 35.3566, 34.4476, 33.5882, 32.6816, 32.0824, 31.0258, 30.6048, 29.4436, 28.7274, 27.957, 27.147, 26.4364, 25.7592, 25.3386, 24.781, 23.8028, 23.656, 22.6544, 21.996, 21.4718, 21.1544, 20.6098, 19.5956, 19.0616, 18.5758, 18.4878, 17.5244, 17.2146, 16.724, 15.8722, 15.5198, 15.0414, 14.941, 14.9048, 13.87, 13.4304, 13.028, 12.4708, 12.37, 12.0624, 11.4668, 11.5532, 11.4352, 11.2564, 10.2744, 10.2118, 9.74720000000002, 10.1456, 9.2928, 8.75040000000001, 8.55279999999999, 8.97899999999998, 8.21019999999999, 8.18340000000001, 7.3494, 7.32499999999999, 7.66140000000001, 6.90300000000002, 7.25439999999998, 6.9042, 7.21499999999997, 6.28640000000001, 6.08139999999997, 6.6764, 6.30099999999999, 5.13900000000001, 5.65800000000002, 5.17320000000001, 4.59019999999998, 4.9538, 5.08280000000002, 4.92200000000003, 4.99020000000002, 4.7328, 5.4538, 4.11360000000002, 4.22340000000003, 4.08780000000002, 3.70800000000003, 4.15559999999999, 4.18520000000001, 3.63720000000001, 3.68220000000002, 3.77960000000002, 3.6078, 2.49160000000001, 3.13099999999997, 2.5376, 3.19880000000001, 3.21100000000001, 2.4502, 3.52820000000003, 2.91199999999998, 3.04480000000001, 2.7432, 2.85239999999999, 2.79880000000003, 2.78579999999999, 1.88679999999999, 2.98860000000002, 2.50639999999999, 1.91239999999999, 2.66160000000002, 2.46820000000002, 1.58199999999999, 1.30399999999997, 2.27379999999999, 2.68939999999998, 1.32900000000001, 3.10599999999999, 1.69080000000002, 2.13740000000001, 2.53219999999999, 1.88479999999998, 1.33240000000001, 1.45119999999997, 1.17899999999997, 2.44119999999998, 1.60659999999996, 2.16700000000003, 0.77940000000001, 2.37900000000002, 2.06700000000001, 1.46000000000004, 2.91160000000002, 1.69200000000001, 0.954600000000028, 2.49300000000005, 2.2722, 1.33500000000004, 2.44899999999996, 1.20140000000004, 3.07380000000001, 2.09739999999999, 2.85640000000001, 2.29960000000005, 2.40899999999999, 1.97040000000004, 0.809799999999996, 1.65279999999996, 2.59979999999996, 0.95799999999997, 2.06799999999998, 2.32780000000002, 4.20159999999998, 1.96320000000003, 1.86400000000003, 1.42999999999995, 3.77940000000001, 1.27200000000005, 1.86440000000005, 2.20600000000002, 3.21900000000005, 1.5154, 2.61019999999996, }, + // precision 8 + (double[]) { 183.2152, 180.2454, 177.2096, 173.6652, 170.6312, 167.6822, 164.249, 161.3296, 158.0038, 155.2074, 152.4612, 149.27, 146.5178, 143.4412, 140.8032, 138.1634, 135.1688, 132.6074, 129.6946, 127.2664, 124.8228, 122.0432, 119.6824, 116.9464, 114.6268, 112.2626, 109.8376, 107.4034, 104.8956, 102.8522, 100.7638, 98.3552, 96.3556, 93.7526, 91.9292, 89.8954, 87.8198, 85.7668, 83.298, 81.6688, 79.9466, 77.9746, 76.1672, 74.3474, 72.3028, 70.8912, 69.114, 67.4646, 65.9744, 64.4092, 62.6022, 60.843, 59.5684, 58.1652, 56.5426, 55.4152, 53.5388, 52.3592, 51.1366, 49.486, 48.3918, 46.5076, 45.509, 44.3834, 43.3498, 42.0668, 40.7346, 40.1228, 38.4528, 37.7, 36.644, 36.0518, 34.5774, 33.9068, 32.432, 32.1666, 30.434, 29.6644, 28.4894, 27.6312, 26.3804, 26.292, 25.5496000000001, 25.0234, 24.8206, 22.6146, 22.4188, 22.117, 20.6762, 20.6576, 19.7864, 19.509, 18.5334, 17.9204, 17.772, 16.2924, 16.8654, 15.1836, 15.745, 15.1316, 15.0386, 14.0136, 13.6342, 12.6196, 12.1866, 12.4281999999999, 11.3324, 10.4794000000001, 11.5038, 10.129, 9.52800000000002, 10.3203999999999, 9.46299999999997, 9.79280000000006, 9.12300000000005, 8.74180000000001, 9.2192, 7.51020000000005, 7.60659999999996, 7.01840000000004, 7.22239999999999, 7.40139999999997, 6.76179999999999, 7.14359999999999, 5.65060000000005, 5.63779999999997, 5.76599999999996, 6.75139999999999, 5.57759999999996, 3.73220000000003, 5.8048, 5.63019999999995, 4.93359999999996, 3.47979999999995, 4.33879999999999, 3.98940000000005, 3.81960000000004, 3.31359999999995, 3.23080000000004, 3.4588, 3.08159999999998, 3.4076, 3.00639999999999, 2.38779999999997, 2.61900000000003, 1.99800000000005, 3.34820000000002, 2.95060000000001, 0.990999999999985, 2.11440000000005, 2.20299999999997, 2.82219999999995, 2.73239999999998, 2.7826, 3.76660000000004, 2.26480000000004, 2.31280000000004, 2.40819999999997, 2.75360000000001, 3.33759999999995, 2.71559999999999, 1.7478000000001, 1.42920000000004, 2.39300000000003, 2.22779999999989, 2.34339999999997, 0.87259999999992, 3.88400000000001, 1.80600000000004, 1.91759999999999, 1.16779999999994, 1.50320000000011, 2.52500000000009, 0.226400000000012, 2.31500000000005, 0.930000000000064, 1.25199999999995, 2.14959999999996, 0.0407999999999902, 2.5447999999999, 1.32960000000003, 0.197400000000016, 2.52620000000002, 3.33279999999991, -1.34300000000007, 0.422199999999975, 0.917200000000093, 1.12920000000008, 1.46060000000011, 1.45779999999991, 2.8728000000001, 3.33359999999993, -1.34079999999994, 1.57680000000005, 0.363000000000056, 1.40740000000005, 0.656600000000026, 0.801400000000058, -0.454600000000028, 1.51919999999996, }, + // precision 9 + (double[]) { 368, 361.8294, 355.2452, 348.6698, 342.1464, 336.2024, 329.8782, 323.6598, 317.462, 311.2826, 305.7102, 299.7416, 293.9366, 288.1046, 282.285, 277.0668, 271.306, 265.8448, 260.301, 254.9886, 250.2422, 244.8138, 239.7074, 234.7428, 229.8402, 225.1664, 220.3534, 215.594, 210.6886, 205.7876, 201.65, 197.228, 192.8036, 188.1666, 184.0818, 180.0824, 176.2574, 172.302, 168.1644, 164.0056, 160.3802, 156.7192, 152.5234, 149.2084, 145.831, 142.485, 139.1112, 135.4764, 131.76, 129.3368, 126.5538, 122.5058, 119.2646, 116.5902, 113.3818, 110.8998, 107.9532, 105.2062, 102.2798, 99.4728, 96.9582, 94.3292, 92.171, 89.7809999999999, 87.5716, 84.7048, 82.5322, 79.875, 78.3972, 75.3464, 73.7274, 71.2834, 70.1444, 68.4263999999999, 66.0166, 64.018, 62.0437999999999, 60.3399999999999, 58.6856, 57.9836, 55.0311999999999, 54.6769999999999, 52.3188, 51.4846, 49.4423999999999, 47.739, 46.1487999999999, 44.9202, 43.4059999999999, 42.5342000000001, 41.2834, 38.8954000000001, 38.3286000000001, 36.2146, 36.6684, 35.9946, 33.123, 33.4338, 31.7378000000001, 29.076, 28.9692, 27.4964, 27.0998, 25.9864, 26.7754, 24.3208, 23.4838, 22.7388000000001, 24.0758000000001, 21.9097999999999, 20.9728, 19.9228000000001, 19.9292, 16.617, 17.05, 18.2996000000001, 15.6128000000001, 15.7392, 14.5174, 13.6322, 12.2583999999999, 13.3766000000001, 11.423, 13.1232, 9.51639999999998, 10.5938000000001, 9.59719999999993, 8.12220000000002, 9.76739999999995, 7.50440000000003, 7.56999999999994, 6.70440000000008, 6.41419999999994, 6.71019999999999, 5.60940000000005, 4.65219999999999, 6.84099999999989, 3.4072000000001, 3.97859999999991, 3.32760000000007, 5.52160000000003, 3.31860000000006, 2.06940000000009, 4.35400000000004, 1.57500000000005, 0.280799999999999, 2.12879999999996, -0.214799999999968, -0.0378000000000611, -0.658200000000079, 0.654800000000023, -0.0697999999999865, 0.858400000000074, -2.52700000000004, -2.1751999999999, -3.35539999999992, -1.04019999999991, -0.651000000000067, -2.14439999999991, -1.96659999999997, -3.97939999999994, -0.604400000000169, -3.08260000000018, -3.39159999999993, -5.29640000000018, -5.38920000000007, -5.08759999999984, -4.69900000000007, -5.23720000000003, -3.15779999999995, -4.97879999999986, -4.89899999999989, -7.48880000000008, -5.94799999999987, -5.68060000000014, -6.67180000000008, -4.70499999999993, -7.27779999999984, -4.6579999999999, -4.4362000000001, -4.32139999999981, -5.18859999999995, -6.66879999999992, -6.48399999999992, -5.1260000000002, -4.4032000000002, -6.13500000000022, -5.80819999999994, -4.16719999999987, -4.15039999999999, -7.45600000000013, -7.24080000000004, -9.83179999999993, -5.80420000000004, -8.6561999999999, -6.99940000000015, -10.5473999999999, -7.34139999999979, -6.80999999999995, -6.29719999999998, -6.23199999999997, }, + // precision 10 + (double[]) { 737.1256, 724.4234, 711.1064, 698.4732, 685.4636, 673.0644, 660.488, 647.9654, 636.0832, 623.7864, 612.1992, 600.2176, 588.5228, 577.1716, 565.7752, 554.899, 543.6126, 532.6492, 521.9474, 511.5214, 501.1064, 490.6364, 480.2468, 470.4588, 460.3832, 451.0584, 440.8606, 431.3868, 422.5062, 413.1862, 404.463, 395.339, 386.1936, 378.1292, 369.1854, 361.2908, 353.3324, 344.8518, 337.5204, 329.4854, 321.9318, 314.552, 306.4658, 299.4256, 292.849, 286.152, 278.8956, 271.8792, 265.118, 258.62, 252.5132, 245.9322, 239.7726, 233.6086, 227.5332, 222.5918, 216.4294, 210.7662, 205.4106, 199.7338, 194.9012, 188.4486, 183.1556, 178.6338, 173.7312, 169.6264, 163.9526, 159.8742, 155.8326, 151.1966, 147.5594, 143.07, 140.037, 134.1804, 131.071, 127.4884, 124.0848, 120.2944, 117.333, 112.9626, 110.2902, 107.0814, 103.0334, 99.4832000000001, 96.3899999999999, 93.7202000000002, 90.1714000000002, 87.2357999999999, 85.9346, 82.8910000000001, 80.0264000000002, 78.3834000000002, 75.1543999999999, 73.8683999999998, 70.9895999999999, 69.4367999999999, 64.8701999999998, 65.0408000000002, 61.6738, 59.5207999999998, 57.0158000000001, 54.2302, 53.0962, 50.4985999999999, 52.2588000000001, 47.3914, 45.6244000000002, 42.8377999999998, 43.0072, 40.6516000000001, 40.2453999999998, 35.2136, 36.4546, 33.7849999999999, 33.2294000000002, 32.4679999999998, 30.8670000000002, 28.6507999999999, 28.9099999999999, 27.5983999999999, 26.1619999999998, 24.5563999999999, 23.2328000000002, 21.9484000000002, 21.5902000000001, 21.3346000000001, 17.7031999999999, 20.6111999999998, 19.5545999999999, 15.7375999999999, 17.0720000000001, 16.9517999999998, 15.326, 13.1817999999998, 14.6925999999999, 13.0859999999998, 13.2754, 10.8697999999999, 11.248, 7.3768, 4.72339999999986, 7.97899999999981, 8.7503999999999, 7.68119999999999, 9.7199999999998, 7.73919999999998, 5.6224000000002, 7.44560000000001, 6.6601999999998, 5.9058, 4.00199999999995, 4.51699999999983, 4.68240000000014, 3.86220000000003, 5.13639999999987, 5.98500000000013, 2.47719999999981, 2.61999999999989, 1.62800000000016, 4.65000000000009, 0.225599999999758, 0.831000000000131, -0.359400000000278, 1.27599999999984, -2.92559999999958, -0.0303999999996449, 2.37079999999969, -2.0033999999996, 0.804600000000391, 0.30199999999968, 1.1247999999996, -2.6880000000001, 0.0321999999996478, -1.18099999999959, -3.9402, -1.47940000000017, -0.188400000000001, -2.10720000000038, -2.04159999999956, -3.12880000000041, -4.16160000000036, -0.612799999999879, -3.48719999999958, -8.17900000000009, -5.37780000000021, -4.01379999999972, -5.58259999999973, -5.73719999999958, -7.66799999999967, -5.69520000000011, -1.1247999999996, -5.58520000000044, -8.04560000000038, -4.64840000000004, -11.6468000000004, -7.97519999999986, -5.78300000000036, -7.67420000000038, -10.6328000000003, -9.81720000000041, }, + // precision 11 + (double[]) { 1476, 1449.6014, 1423.5802, 1397.7942, 1372.3042, 1347.2062, 1321.8402, 1297.2292, 1272.9462, 1248.9926, 1225.3026, 1201.4252, 1178.0578, 1155.6092, 1132.626, 1110.5568, 1088.527, 1066.5154, 1045.1874, 1024.3878, 1003.37, 982.1972, 962.5728, 942.1012, 922.9668, 903.292, 884.0772, 864.8578, 846.6562, 828.041, 809.714, 792.3112, 775.1806, 757.9854, 740.656, 724.346, 707.5154, 691.8378, 675.7448, 659.6722, 645.5722, 630.1462, 614.4124, 600.8728, 585.898, 572.408, 558.4926, 544.4938, 531.6776, 517.282, 505.7704, 493.1012, 480.7388, 467.6876, 456.1872, 445.5048, 433.0214, 420.806, 411.409, 400.4144, 389.4294, 379.2286, 369.651, 360.6156, 350.337, 342.083, 332.1538, 322.5094, 315.01, 305.6686, 298.1678, 287.8116, 280.9978, 271.9204, 265.3286, 257.5706, 249.6014, 242.544, 235.5976, 229.583, 220.9438, 214.672, 208.2786, 201.8628, 195.1834, 191.505, 186.1816, 178.5188, 172.2294, 167.8908, 161.0194, 158.052, 151.4588, 148.1596, 143.4344, 138.5238, 133.13, 127.6374, 124.8162, 118.7894, 117.3984, 114.6078, 109.0858, 105.1036, 103.6258, 98.6018000000004, 95.7618000000002, 93.5821999999998, 88.5900000000001, 86.9992000000002, 82.8800000000001, 80.4539999999997, 74.6981999999998, 74.3644000000004, 73.2914000000001, 65.5709999999999, 66.9232000000002, 65.1913999999997, 62.5882000000001, 61.5702000000001, 55.7035999999998, 56.1764000000003, 52.7596000000003, 53.0302000000001, 49.0609999999997, 48.4694, 44.933, 46.0474000000004, 44.7165999999997, 41.9416000000001, 39.9207999999999, 35.6328000000003, 35.5276000000003, 33.1934000000001, 33.2371999999996, 33.3864000000003, 33.9228000000003, 30.2371999999996, 29.1373999999996, 25.2272000000003, 24.2942000000003, 19.8338000000003, 18.9005999999999, 23.0907999999999, 21.8544000000002, 19.5176000000001, 15.4147999999996, 16.9314000000004, 18.6737999999996, 12.9877999999999, 14.3688000000002, 12.0447999999997, 15.5219999999999, 12.5299999999997, 14.5940000000001, 14.3131999999996, 9.45499999999993, 12.9441999999999, 3.91139999999996, 13.1373999999996, 5.44720000000052, 9.82779999999912, 7.87279999999919, 3.67760000000089, 5.46980000000076, 5.55099999999948, 5.65979999999945, 3.89439999999922, 3.1275999999998, 5.65140000000065, 6.3062000000009, 3.90799999999945, 1.87060000000019, 5.17020000000048, 2.46680000000015, 0.770000000000437, -3.72340000000077, 1.16400000000067, 8.05340000000069, 0.135399999999208, 2.15940000000046, 0.766999999999825, 1.0594000000001, 3.15500000000065, -0.287399999999252, 2.37219999999979, -2.86620000000039, -1.63199999999961, -2.22979999999916, -0.15519999999924, -1.46039999999994, -0.262199999999211, -2.34460000000036, -2.8078000000005, -3.22179999999935, -5.60159999999996, -8.42200000000048, -9.43740000000071, 0.161799999999857, -10.4755999999998, -10.0823999999993, }, + // precision 12 + (double[]) { 2953, 2900.4782, 2848.3568, 2796.3666, 2745.324, 2694.9598, 2644.648, 2595.539, 2546.1474, 2498.2576, 2450.8376, 2403.6076, 2357.451, 2311.38, 2266.4104, 2221.5638, 2176.9676, 2134.193, 2090.838, 2048.8548, 2007.018, 1966.1742, 1925.4482, 1885.1294, 1846.4776, 1807.4044, 1768.8724, 1731.3732, 1693.4304, 1657.5326, 1621.949, 1586.5532, 1551.7256, 1517.6182, 1483.5186, 1450.4528, 1417.865, 1385.7164, 1352.6828, 1322.6708, 1291.8312, 1260.9036, 1231.476, 1201.8652, 1173.6718, 1145.757, 1119.2072, 1092.2828, 1065.0434, 1038.6264, 1014.3192, 988.5746, 965.0816, 940.1176, 917.9796, 894.5576, 871.1858, 849.9144, 827.1142, 805.0818, 783.9664, 763.9096, 742.0816, 724.3962, 706.3454, 688.018, 667.4214, 650.3106, 633.0686, 613.8094, 597.818, 581.4248, 563.834, 547.363, 531.5066, 520.455400000001, 505.583199999999, 488.366, 476.480799999999, 459.7682, 450.0522, 434.328799999999, 423.952799999999, 408.727000000001, 399.079400000001, 387.252200000001, 373.987999999999, 360.852000000001, 351.6394, 339.642, 330.902400000001, 322.661599999999, 311.662200000001, 301.3254, 291.7484, 279.939200000001, 276.7508, 263.215200000001, 254.811400000001, 245.5494, 242.306399999999, 234.8734, 223.787200000001, 217.7156, 212.0196, 200.793, 195.9748, 189.0702, 182.449199999999, 177.2772, 170.2336, 164.741, 158.613600000001, 155.311, 147.5964, 142.837, 137.3724, 132.0162, 130.0424, 121.9804, 120.451800000001, 114.8968, 111.585999999999, 105.933199999999, 101.705, 98.5141999999996, 95.0488000000005, 89.7880000000005, 91.4750000000004, 83.7764000000006, 80.9698000000008, 72.8574000000008, 73.1615999999995, 67.5838000000003, 62.6263999999992, 63.2638000000006, 66.0977999999996, 52.0843999999997, 58.9956000000002, 47.0912000000008, 46.4956000000002, 48.4383999999991, 47.1082000000006, 43.2392, 37.2759999999998, 40.0283999999992, 35.1864000000005, 35.8595999999998, 32.0998, 28.027, 23.6694000000007, 33.8266000000003, 26.3736000000008, 27.2008000000005, 21.3245999999999, 26.4115999999995, 23.4521999999997, 19.5013999999992, 19.8513999999996, 10.7492000000002, 18.6424000000006, 13.1265999999996, 18.2436000000016, 6.71860000000015, 3.39459999999963, 6.33759999999893, 7.76719999999841, 0.813999999998487, 3.82819999999992, 0.826199999999517, 8.07440000000133, -1.59080000000176, 5.01780000000144, 0.455399999998917, -0.24199999999837, 0.174800000000687, -9.07640000000174, -4.20160000000033, -3.77520000000004, -4.75179999999818, -5.3724000000002, -8.90680000000066, -6.10239999999976, -5.74120000000039, -9.95339999999851, -3.86339999999836, -13.7304000000004, -16.2710000000006, -7.51359999999841, -3.30679999999847, -13.1339999999982, -10.0551999999989, -6.72019999999975, -8.59660000000076, -10.9307999999983, -1.8775999999998, -4.82259999999951, -13.7788, -21.6470000000008, -10.6735999999983, -15.7799999999988, }, + // precision 13 + (double[]) { 5907.5052, 5802.2672, 5697.347, 5593.5794, 5491.2622, 5390.5514, 5290.3376, 5191.6952, 5093.5988, 4997.3552, 4902.5972, 4808.3082, 4715.5646, 4624.109, 4533.8216, 4444.4344, 4356.3802, 4269.2962, 4183.3784, 4098.292, 4014.79, 3932.4574, 3850.6036, 3771.2712, 3691.7708, 3615.099, 3538.1858, 3463.4746, 3388.8496, 3315.6794, 3244.5448, 3173.7516, 3103.3106, 3033.6094, 2966.5642, 2900.794, 2833.7256, 2769.81, 2707.3196, 2644.0778, 2583.9916, 2523.4662, 2464.124, 2406.073, 2347.0362, 2292.1006, 2238.1716, 2182.7514, 2128.4884, 2077.1314, 2025.037, 1975.3756, 1928.933, 1879.311, 1831.0006, 1783.2144, 1738.3096, 1694.5144, 1649.024, 1606.847, 1564.7528, 1525.3168, 1482.5372, 1443.9668, 1406.5074, 1365.867, 1329.2186, 1295.4186, 1257.9716, 1225.339, 1193.2972, 1156.3578, 1125.8686, 1091.187, 1061.4094, 1029.4188, 1000.9126, 972.3272, 944.004199999999, 915.7592, 889.965, 862.834200000001, 840.4254, 812.598399999999, 785.924200000001, 763.050999999999, 741.793799999999, 721.466, 699.040799999999, 677.997200000002, 649.866999999998, 634.911800000002, 609.8694, 591.981599999999, 570.2922, 557.129199999999, 538.3858, 521.872599999999, 502.951400000002, 495.776399999999, 475.171399999999, 459.751, 439.995200000001, 426.708999999999, 413.7016, 402.3868, 387.262599999998, 372.0524, 357.050999999999, 342.5098, 334.849200000001, 322.529399999999, 311.613799999999, 295.848000000002, 289.273000000001, 274.093000000001, 263.329600000001, 251.389599999999, 245.7392, 231.9614, 229.7952, 217.155200000001, 208.9588, 199.016599999999, 190.839199999999, 180.6976, 176.272799999999, 166.976999999999, 162.5252, 151.196400000001, 149.386999999999, 133.981199999998, 130.0586, 130.164000000001, 122.053400000001, 110.7428, 108.1276, 106.232400000001, 100.381600000001, 98.7668000000012, 86.6440000000002, 79.9768000000004, 82.4722000000002, 68.7026000000005, 70.1186000000016, 71.9948000000004, 58.998599999999, 59.0492000000013, 56.9818000000014, 47.5338000000011, 42.9928, 51.1591999999982, 37.2740000000013, 42.7220000000016, 31.3734000000004, 26.8090000000011, 25.8934000000008, 26.5286000000015, 29.5442000000003, 19.3503999999994, 26.0760000000009, 17.9527999999991, 14.8419999999969, 10.4683999999979, 8.65899999999965, 9.86720000000059, 4.34139999999752, -0.907800000000861, -3.32080000000133, -0.936199999996461, -11.9916000000012, -8.87000000000262, -6.33099999999831, -11.3366000000024, -15.9207999999999, -9.34659999999712, -15.5034000000014, -19.2097999999969, -15.357799999998, -28.2235999999975, -30.6898000000001, -19.3271999999997, -25.6083999999973, -24.409599999999, -13.6385999999984, -33.4473999999973, -32.6949999999997, -28.9063999999998, -31.7483999999968, -32.2935999999972, -35.8329999999987, -47.620600000002, -39.0855999999985, -33.1434000000008, -46.1371999999974, -37.5892000000022, -46.8164000000033, -47.3142000000007, -60.2914000000019, -37.7575999999972, }, + // precision 14 + (double[]) { 11816.475, 11605.0046, 11395.3792, 11188.7504, 10984.1814, 10782.0086, 10582.0072, 10384.503, 10189.178, 9996.2738, 9806.0344, 9617.9798, 9431.394, 9248.7784, 9067.6894, 8889.6824, 8712.9134, 8538.8624, 8368.4944, 8197.7956, 8031.8916, 7866.6316, 7703.733, 7544.5726, 7386.204, 7230.666, 7077.8516, 6926.7886, 6778.6902, 6631.9632, 6487.304, 6346.7486, 6206.4408, 6070.202, 5935.2576, 5799.924, 5671.0324, 5541.9788, 5414.6112, 5290.0274, 5166.723, 5047.6906, 4929.162, 4815.1406, 4699.127, 4588.5606, 4477.7394, 4369.4014, 4264.2728, 4155.9224, 4055.581, 3955.505, 3856.9618, 3761.3828, 3666.9702, 3575.7764, 3482.4132, 3395.0186, 3305.8852, 3221.415, 3138.6024, 3056.296, 2970.4494, 2896.1526, 2816.8008, 2740.2156, 2670.497, 2594.1458, 2527.111, 2460.8168, 2387.5114, 2322.9498, 2260.6752, 2194.2686, 2133.7792, 2074.767, 2015.204, 1959.4226, 1898.6502, 1850.006, 1792.849, 1741.4838, 1687.9778, 1638.1322, 1589.3266, 1543.1394, 1496.8266, 1447.8516, 1402.7354, 1361.9606, 1327.0692, 1285.4106, 1241.8112, 1201.6726, 1161.973, 1130.261, 1094.2036, 1048.2036, 1020.6436, 990.901400000002, 961.199800000002, 924.769800000002, 899.526400000002, 872.346400000002, 834.375, 810.432000000001, 780.659800000001, 756.013800000001, 733.479399999997, 707.923999999999, 673.858, 652.222399999999, 636.572399999997, 615.738599999997, 586.696400000001, 564.147199999999, 541.679600000003, 523.943599999999, 505.714599999999, 475.729599999999, 461.779600000002, 449.750800000002, 439.020799999998, 412.7886, 400.245600000002, 383.188199999997, 362.079599999997, 357.533799999997, 334.319000000003, 327.553399999997, 308.559399999998, 291.270199999999, 279.351999999999, 271.791400000002, 252.576999999997, 247.482400000001, 236.174800000001, 218.774599999997, 220.155200000001, 208.794399999999, 201.223599999998, 182.995600000002, 185.5268, 164.547400000003, 176.5962, 150.689599999998, 157.8004, 138.378799999999, 134.021200000003, 117.614399999999, 108.194000000003, 97.0696000000025, 89.6042000000016, 95.6030000000028, 84.7810000000027, 72.635000000002, 77.3482000000004, 59.4907999999996, 55.5875999999989, 50.7346000000034, 61.3916000000027, 50.9149999999936, 39.0384000000049, 58.9395999999979, 29.633600000001, 28.2032000000036, 26.0078000000067, 17.0387999999948, 9.22000000000116, 13.8387999999977, 8.07240000000456, 14.1549999999988, 15.3570000000036, 3.42660000000615, 6.24820000000182, -2.96940000000177, -8.79940000000352, -5.97860000000219, -14.4048000000039, -3.4143999999942, -13.0148000000045, -11.6977999999945, -25.7878000000055, -22.3185999999987, -24.409599999999, -31.9756000000052, -18.9722000000038, -22.8678000000073, -30.8972000000067, -32.3715999999986, -22.3907999999938, -43.6720000000059, -35.9038, -39.7492000000057, -54.1641999999993, -45.2749999999942, -42.2989999999991, -44.1089999999967, -64.3564000000042, -49.9551999999967, -42.6116000000038, }, + // precision 15 + (double[]) { 23634.0036, 23210.8034, 22792.4744, 22379.1524, 21969.7928, 21565.326, 21165.3532, 20770.2806, 20379.9892, 19994.7098, 19613.318, 19236.799, 18865.4382, 18498.8244, 18136.5138, 17778.8668, 17426.2344, 17079.32, 16734.778, 16397.2418, 16063.3324, 15734.0232, 15409.731, 15088.728, 14772.9896, 14464.1402, 14157.5588, 13855.5958, 13559.3296, 13264.9096, 12978.326, 12692.0826, 12413.8816, 12137.3192, 11870.2326, 11602.5554, 11340.3142, 11079.613, 10829.5908, 10583.5466, 10334.0344, 10095.5072, 9859.694, 9625.2822, 9395.7862, 9174.0586, 8957.3164, 8738.064, 8524.155, 8313.7396, 8116.9168, 7913.542, 7718.4778, 7521.65, 7335.5596, 7154.2906, 6968.7396, 6786.3996, 6613.236, 6437.406, 6270.6598, 6107.7958, 5945.7174, 5787.6784, 5635.5784, 5482.308, 5337.9784, 5190.0864, 5045.9158, 4919.1386, 4771.817, 4645.7742, 4518.4774, 4385.5454, 4262.6622, 4142.74679999999, 4015.5318, 3897.9276, 3790.7764, 3685.13800000001, 3573.6274, 3467.9706, 3368.61079999999, 3271.5202, 3170.3848, 3076.4656, 2982.38400000001, 2888.4664, 2806.4868, 2711.9564, 2634.1434, 2551.3204, 2469.7662, 2396.61139999999, 2318.9902, 2243.8658, 2171.9246, 2105.01360000001, 2028.8536, 1960.9952, 1901.4096, 1841.86079999999, 1777.54700000001, 1714.5802, 1654.65059999999, 1596.311, 1546.2016, 1492.3296, 1433.8974, 1383.84600000001, 1339.4152, 1293.5518, 1245.8686, 1193.50659999999, 1162.27959999999, 1107.19439999999, 1069.18060000001, 1035.09179999999, 999.679000000004, 957.679999999993, 925.300199999998, 888.099400000006, 848.638600000006, 818.156400000007, 796.748399999997, 752.139200000005, 725.271200000003, 692.216, 671.633600000001, 647.939799999993, 621.670599999998, 575.398799999995, 561.226599999995, 532.237999999998, 521.787599999996, 483.095799999996, 467.049599999998, 465.286399999997, 415.548599999995, 401.047399999996, 380.607999999993, 377.362599999993, 347.258799999996, 338.371599999999, 310.096999999994, 301.409199999995, 276.280799999993, 265.586800000005, 258.994399999996, 223.915999999997, 215.925399999993, 213.503800000006, 191.045400000003, 166.718200000003, 166.259000000005, 162.941200000001, 148.829400000002, 141.645999999993, 123.535399999993, 122.329800000007, 89.473399999988, 80.1962000000058, 77.5457999999926, 59.1056000000099, 83.3509999999951, 52.2906000000075, 36.3979999999865, 40.6558000000077, 42.0003999999899, 19.6630000000005, 19.7153999999864, -8.38539999999921, -0.692799999989802, 0.854800000000978, 3.23219999999856, -3.89040000000386, -5.25880000001052, -24.9052000000083, -22.6837999999989, -26.4286000000138, -34.997000000003, -37.0216000000073, -43.430400000012, -58.2390000000014, -68.8034000000043, -56.9245999999985, -57.8583999999973, -77.3097999999882, -73.2793999999994, -81.0738000000129, -87.4530000000086, -65.0254000000132, -57.296399999992, -96.2746000000043, -103.25, -96.081600000005, -91.5542000000132, -102.465200000006, -107.688599999994, -101.458000000013, -109.715800000005, }, + // precision 16 + (double[]) { 47270, 46423.3584, 45585.7074, 44757.152, 43938.8416, 43130.9514, 42330.03, 41540.407, 40759.6348, 39988.206, 39226.5144, 38473.2096, 37729.795, 36997.268, 36272.6448, 35558.665, 34853.0248, 34157.4472, 33470.5204, 32793.5742, 32127.0194, 31469.4182, 30817.6136, 30178.6968, 29546.8908, 28922.8544, 28312.271, 27707.0924, 27114.0326, 26526.692, 25948.6336, 25383.7826, 24823.5998, 24272.2974, 23732.2572, 23201.4976, 22674.2796, 22163.6336, 21656.515, 21161.7362, 20669.9368, 20189.4424, 19717.3358, 19256.3744, 18795.9638, 18352.197, 17908.5738, 17474.391, 17052.918, 16637.2236, 16228.4602, 15823.3474, 15428.6974, 15043.0284, 14667.6278, 14297.4588, 13935.2882, 13578.5402, 13234.6032, 12882.1578, 12548.0728, 12219.231, 11898.0072, 11587.2626, 11279.9072, 10973.5048, 10678.5186, 10392.4876, 10105.2556, 9825.766, 9562.5444, 9294.2222, 9038.2352, 8784.848, 8533.2644, 8301.7776, 8058.30859999999, 7822.94579999999, 7599.11319999999, 7366.90779999999, 7161.217, 6957.53080000001, 6736.212, 6548.21220000001, 6343.06839999999, 6156.28719999999, 5975.15419999999, 5791.75719999999, 5621.32019999999, 5451.66, 5287.61040000001, 5118.09479999999, 4957.288, 4798.4246, 4662.17559999999, 4512.05900000001, 4364.68539999999, 4220.77720000001, 4082.67259999999, 3957.19519999999, 3842.15779999999, 3699.3328, 3583.01180000001, 3473.8964, 3338.66639999999, 3233.55559999999, 3117.799, 3008.111, 2909.69140000001, 2814.86499999999, 2719.46119999999, 2624.742, 2532.46979999999, 2444.7886, 2370.1868, 2272.45259999999, 2196.19260000001, 2117.90419999999, 2023.2972, 1969.76819999999, 1885.58979999999, 1833.2824, 1733.91200000001, 1682.54920000001, 1604.57980000001, 1556.11240000001, 1491.3064, 1421.71960000001, 1371.22899999999, 1322.1324, 1264.7892, 1196.23920000001, 1143.8474, 1088.67240000001, 1073.60380000001, 1023.11660000001, 959.036400000012, 927.433199999999, 906.792799999996, 853.433599999989, 841.873800000001, 791.1054, 756.899999999994, 704.343200000003, 672.495599999995, 622.790399999998, 611.254799999995, 567.283200000005, 519.406599999988, 519.188400000014, 495.312800000014, 451.350799999986, 443.973399999988, 431.882199999993, 392.027000000002, 380.924200000009, 345.128999999986, 298.901400000002, 287.771999999997, 272.625, 247.253000000026, 222.490600000019, 223.590000000026, 196.407599999977, 176.425999999978, 134.725199999986, 132.4804, 110.445599999977, 86.7939999999944, 56.7038000000175, 64.915399999998, 38.3726000000024, 37.1606000000029, 46.170999999973, 49.1716000000015, 15.3362000000197, 6.71639999997569, -34.8185999999987, -39.4476000000141, 12.6830000000191, -12.3331999999937, -50.6565999999875, -59.9538000000175, -65.1054000000004, -70.7576000000117, -106.325200000021, -126.852200000023, -110.227599999984, -132.885999999999, -113.897200000007, -142.713800000027, -151.145399999979, -150.799200000009, -177.756200000003, -156.036399999983, -182.735199999996, -177.259399999981, -198.663600000029, -174.577600000019, -193.84580000001, }, + // precision 17 + (double[]) { 94541, 92848.811, 91174.019, 89517.558, 87879.9705, 86262.7565, 84663.5125, 83083.7435, 81521.7865, 79977.272, 78455.9465, 76950.219, 75465.432, 73994.152, 72546.71, 71115.2345, 69705.6765, 68314.937, 66944.2705, 65591.255, 64252.9485, 62938.016, 61636.8225, 60355.592, 59092.789, 57850.568, 56624.518, 55417.343, 54231.1415, 53067.387, 51903.526, 50774.649, 49657.6415, 48561.05, 47475.7575, 46410.159, 45364.852, 44327.053, 43318.4005, 42325.6165, 41348.4595, 40383.6265, 39436.77, 38509.502, 37594.035, 36695.939, 35818.6895, 34955.691, 34115.8095, 33293.949, 32465.0775, 31657.6715, 30877.2585, 30093.78, 29351.3695, 28594.1365, 27872.115, 27168.7465, 26477.076, 25774.541, 25106.5375, 24452.5135, 23815.5125, 23174.0655, 22555.2685, 21960.2065, 21376.3555, 20785.1925, 20211.517, 19657.0725, 19141.6865, 18579.737, 18081.3955, 17578.995, 17073.44, 16608.335, 16119.911, 15651.266, 15194.583, 14749.0495, 14343.4835, 13925.639, 13504.509, 13099.3885, 12691.2855, 12328.018, 11969.0345, 11596.5145, 11245.6355, 10917.6575, 10580.9785, 10277.8605, 9926.58100000001, 9605.538, 9300.42950000003, 8989.97850000003, 8728.73249999998, 8448.3235, 8175.31050000002, 7898.98700000002, 7629.79100000003, 7413.76199999999, 7149.92300000001, 6921.12650000001, 6677.1545, 6443.28000000003, 6278.23450000002, 6014.20049999998, 5791.20299999998, 5605.78450000001, 5438.48800000001, 5234.2255, 5059.6825, 4887.43349999998, 4682.935, 4496.31099999999, 4322.52250000002, 4191.42499999999, 4021.24200000003, 3900.64799999999, 3762.84250000003, 3609.98050000001, 3502.29599999997, 3363.84250000003, 3206.54849999998, 3079.70000000001, 2971.42300000001, 2867.80349999998, 2727.08100000001, 2630.74900000001, 2496.6165, 2440.902, 2356.19150000002, 2235.58199999999, 2120.54149999999, 2012.25449999998, 1933.35600000003, 1820.93099999998, 1761.54800000001, 1663.09350000002, 1578.84600000002, 1509.48149999999, 1427.3345, 1379.56150000001, 1306.68099999998, 1212.63449999999, 1084.17300000001, 1124.16450000001, 1060.69949999999, 1007.48849999998, 941.194499999983, 879.880500000028, 836.007500000007, 782.802000000025, 748.385499999975, 647.991500000004, 626.730500000005, 570.776000000013, 484.000500000024, 513.98550000001, 418.985499999952, 386.996999999974, 370.026500000036, 355.496999999974, 356.731499999994, 255.92200000002, 259.094000000041, 205.434499999974, 165.374500000034, 197.347500000033, 95.718499999959, 67.6165000000037, 54.6970000000438, 31.7395000000251, -15.8784999999916, 8.42500000004657, -26.3754999999655, -118.425500000012, -66.6629999999423, -42.9745000000112, -107.364999999991, -189.839000000036, -162.611499999999, -164.964999999967, -189.079999999958, -223.931499999948, -235.329999999958, -269.639500000048, -249.087999999989, -206.475499999942, -283.04449999996, -290.667000000016, -304.561499999953, -336.784499999951, -380.386500000022, -283.280499999993, -364.533000000054, -389.059499999974, -364.454000000027, -415.748000000021, -417.155000000028, }, + // precision 18 + (double[]) { 189083, 185696.913, 182348.774, 179035.946, 175762.762, 172526.444, 169329.754, 166166.099, 163043.269, 159958.91, 156907.912, 153906.845, 150924.199, 147996.568, 145093.457, 142239.233, 139421.475, 136632.27, 133889.588, 131174.2, 128511.619, 125868.621, 123265.385, 120721.061, 118181.769, 115709.456, 113252.446, 110840.198, 108465.099, 106126.164, 103823.469, 101556.618, 99308.004, 97124.508, 94937.803, 92833.731, 90745.061, 88677.627, 86617.47, 84650.442, 82697.833, 80769.132, 78879.629, 77014.432, 75215.626, 73384.587, 71652.482, 69895.93, 68209.301, 66553.669, 64921.981, 63310.323, 61742.115, 60205.018, 58698.658, 57190.657, 55760.865, 54331.169, 52908.167, 51550.273, 50225.254, 48922.421, 47614.533, 46362.049, 45098.569, 43926.083, 42736.03, 41593.473, 40425.26, 39316.237, 38243.651, 37170.617, 36114.609, 35084.19, 34117.233, 33206.509, 32231.505, 31318.728, 30403.404, 29540.0550000001, 28679.236, 27825.862, 26965.216, 26179.148, 25462.08, 24645.952, 23922.523, 23198.144, 22529.128, 21762.4179999999, 21134.779, 20459.117, 19840.818, 19187.04, 18636.3689999999, 17982.831, 17439.7389999999, 16874.547, 16358.2169999999, 15835.684, 15352.914, 14823.681, 14329.313, 13816.897, 13342.874, 12880.882, 12491.648, 12021.254, 11625.392, 11293.7610000001, 10813.697, 10456.209, 10099.074, 9755.39000000001, 9393.18500000006, 9047.57900000003, 8657.98499999999, 8395.85900000005, 8033, 7736.95900000003, 7430.59699999995, 7258.47699999996, 6924.58200000005, 6691.29399999999, 6357.92500000005, 6202.05700000003, 5921.19700000004, 5628.28399999999, 5404.96799999999, 5226.71100000001, 4990.75600000005, 4799.77399999998, 4622.93099999998, 4472.478, 4171.78700000001, 3957.46299999999, 3868.95200000005, 3691.14300000004, 3474.63100000005, 3341.67200000002, 3109.14000000001, 3071.97400000005, 2796.40399999998, 2756.17799999996, 2611.46999999997, 2471.93000000005, 2382.26399999997, 2209.22400000005, 2142.28399999999, 2013.96100000001, 1911.18999999994, 1818.27099999995, 1668.47900000005, 1519.65800000005, 1469.67599999998, 1367.13800000004, 1248.52899999998, 1181.23600000003, 1022.71900000004, 1088.20700000005, 959.03600000008, 876.095999999903, 791.183999999892, 703.337000000058, 731.949999999953, 586.86400000006, 526.024999999907, 323.004999999888, 320.448000000091, 340.672999999952, 309.638999999966, 216.601999999955, 102.922999999952, 19.2399999999907, -0.114000000059605, -32.6240000000689, -89.3179999999702, -153.497999999905, -64.2970000000205, -143.695999999996, -259.497999999905, -253.017999999924, -213.948000000091, -397.590000000084, -434.006000000052, -403.475000000093, -297.958000000101, -404.317000000039, -528.898999999976, -506.621000000043, -513.205000000075, -479.351000000024, -596.139999999898, -527.016999999993, -664.681000000099, -680.306000000099, -704.050000000047, -850.486000000034, -757.43200000003, -713.308999999892, } +}; +/* + * Returns the bias correctors from the + * hyperloglog paper + */ +static double alpha(unsigned char precision) { + switch (precision) { + case 4: + return 0.673; + case 5: + return 0.697; + case 6: + return 0.709; + default: + return 0.7213 / (1 + 1.079 / NUM_REG(precision)); + } +} + +// Estimates cardinality using a linear counting. Used when some registers still have a zero value. +static double linear_count(int num_reg, int num_zero) +{ + double est = num_reg * log((double)num_reg / (double)num_zero); + return est; +} +/** + * Binary searches for the nearest matching index + * @return The matching index, or closest match + */ +static int binary_search(double val, int num, const double *array) { + int low=0, mid, high=num-1; + while (low < high) { + mid = (low + high) / 2; + if (val > array[mid]) { + low = mid + 1; + } else if (val == array[mid]) { + return mid; + } else { + high = mid - 1; + } + } + return low; +} +/** + * Interpolates the bias estimate using the + * empircal data collected by Google, from the + * paper mentioned above. + */ +static double bias_estimate(int precision, double raw_est) { + // Determine the samples available + int samples; + switch (precision) { + case 4: + samples = 80; + break; + case 5: + samples = 160; + break; + default: + samples = 200; + break; + } + + // Get the proper arrays based on precision + const double *estimates = *(g_rawEstimateData+(precision-4)); + const double *biases = *(g_biasData+(precision-4)); + + // Get the matching biases + int idx = binary_search(raw_est, samples, estimates); + if (idx == 0) + return biases[0]; + else if (idx == samples) + return biases[samples-1]; + else + return (biases[idx] + biases[idx-1]) / 2; +} + +// Computes the raw cardinality estimate +static double hll_raw_estimate(const uint32_t *registers, unsigned char precision, long long reset_idx, long long time_window_ms, int *num_zero) +{ + int m = NUM_REG(precision); + + double Mi=0, Wi=0;//Wi as the time window span of the i-th register Mi. + double lambda=0; + double harmonic_mean=0; + *num_zero=0; + int effective_reg=0; + for (int i=0; i < m; i++) + { + Wi=(i>=reset_idx)?(m+reset_idx-i):(reset_idx-i)+1; + Mi = get_register(registers, i); + if(!Mi) + *num_zero += 1; + //STHLL++ neglects the 1/8 of the lowest registers to reduce the variance in the final evaluation. + if(Wi < m/8 && time_window_ms) + { + continue; + } + effective_reg++; + lambda=pow(2.0, Mi-1)*m*m/Wi; + harmonic_mean += 1/lambda; + } + harmonic_mean = effective_reg/harmonic_mean; + return harmonic_mean; +} +double hll_count(const uint32_t *registers, unsigned char precision, long long reset_idx, long long time_window_ms) +{ + int num_zero=0; + double raw_est=hll_raw_estimate(registers, precision, reset_idx, time_window_ms, &num_zero); + raw_est*=alpha(precision); + double est=0; + double num_reg=NUM_REG(precision); + if (raw_est <= 5 * num_reg) { + raw_est -= bias_estimate(precision, raw_est); + } + if(raw_est<=5*num_reg/2) + { + if(num_zero) + { + est=linear_count(num_reg, num_zero); + assert(est>=0); + return est; + } + } + if(raw_est>INT32_MAX/30) + { + return INT32_MIN*log(1-raw_est/INT32_MAX); + } + assert(raw_est>=0); + return raw_est; +} +void hll_reset_registers(uint32_t *registers, unsigned char precision, long long offset, long long count) +{ + int num_reg=NUM_REG(precision); + if(count>=num_reg) + { + clear_register(registers, precision); + return; + } + for(int i=0; i<count; i++) + { + reset_register(registers, (offset+i)%num_reg); + } + return; +} +void hll_advance_reset_index(long long *reset_idx, long long reset_reg_count, unsigned char precision) +{ + *reset_idx=(*reset_idx+reset_reg_count)%NUM_REG(precision); + return; +} +long long hll_get_reset_register_count(unsigned char precision, long long time_window_ms, struct timeval now, struct timeval *reset_time) +{ + long long reset_time_slot_us=RESET_TIME_SLOT_US(time_window_ms, precision); + long long delta_us=timeval_delta_us((*reset_time), now); + if(delta_us < reset_time_slot_us) + return 0; + + //We cannot use 'now' as reset_time, but rounded to the nearest time slot. + long long reset_bit_count=0; + reset_bit_count=delta_us/reset_time_slot_us; + struct timeval elapse; + elapse.tv_sec=reset_time_slot_us*reset_bit_count/1000/1000; + elapse.tv_usec=reset_time_slot_us*reset_bit_count%(1000*1000); + timeradd(reset_time, &elapse, reset_time); + return reset_bit_count; +} +void hll_merge(uint32_t *dst_registers, const uint32_t *src_registers, unsigned char precision) +{ + int n_register=NUM_REG(precision); + int s_reg=0, d_reg=0; + for(int i=0; i<n_register; i++) + { + + s_reg=get_register(src_registers, i); + d_reg=get_register(dst_registers, i); + set_register(dst_registers, i, MAX(s_reg, d_reg)); + } +} +double hll_error(unsigned char precision) +{ + /* + * Error of HLL is 1.04 / sqrt(m) + * m is given by 2^p + */ + return 1.04/sqrt(NUM_REG(precision)); +} +size_t hll_size(unsigned char precision) +{ + // Determine how many registers are needed + int num_reg = NUM_REG(precision); + + // Get the full words required + int words = INT_CEIL(num_reg, REG_PER_WORD); + return sizeof(uint32_t)*words; +} +int hll_should_slide(unsigned char precision, long long time_window_ms, struct timeval now, struct timeval reset_time) +{ + assert(time_window_ms>0); + long long reset_time_slot_us=RESET_TIME_SLOT_US(time_window_ms, precision); + long long delta_us=timeval_delta_us(reset_time, now); + if(delta_us >= reset_time_slot_us) + return 1; + return 0; +} + +void hll_encode_into_base64(const uint32_t *registers, unsigned char precision, char **blob, size_t *blob_sz) +{ + size_t sz=0, offset=0; + size_t num_reg = NUM_REG(precision); + size_t words = INT_CEIL(num_reg, REG_PER_WORD); + + sz = hll_size(precision) + 2; // [version][precision][data..., + 2 for version and precision + unsigned char *buffer = ALLOC(unsigned char, sz); + + const unsigned char version = 1; + memcpy(buffer+offset, &version, 1); + offset += 1; + + memcpy(buffer+offset, &precision, 1); + offset += 1; + + for (int i = 0; i < words; i++) { + uint32_t word = registers[i]; + buffer[offset++] = word >> 24; + buffer[offset++] = word >> 16; + buffer[offset++] = word >> 8; + buffer[offset++] = word; + } + + char *enc = b64_encode(buffer, sz); + free(buffer); + *blob_sz = strlen(enc); + *blob = enc; + return; +}
\ No newline at end of file diff --git a/src/cells/hll_common.h b/src/cells/hll_common.h new file mode 100644 index 0000000..b62530a --- /dev/null +++ b/src/cells/hll_common.h @@ -0,0 +1,28 @@ +#pragma once + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include <stdint.h> +#include <sys/time.h>//gettimeofday +#include <stddef.h> + +uint32_t *hll_create_register(int precision); +void hll_free_register(uint32_t *registers); +uint32_t *hll_duplicate(const uint32_t *registers, unsigned char precision); +int hll_add_hash(uint32_t *registers, unsigned char precision, uint64_t hash); +double hll_count(const uint32_t *registers, unsigned char precision, long long reset_idx, long long time_window_ms); +size_t hll_size(unsigned char precision); +double hll_error(unsigned char precision); +int hll_should_slide(unsigned char precision, long long time_window_ms, struct timeval now, struct timeval reset_time); +void hll_reset_registers(uint32_t *registers, unsigned char precision, long long offset, long long count); +long long hll_get_reset_register_count(unsigned char precision, long long time_window_ms, struct timeval now, struct timeval *reset_time); +void hll_advance_reset_index(long long *reset_idx, long long reset_reg_count, unsigned char precision); +void hll_merge(uint32_t *dst_registers, const uint32_t *src_registers, unsigned char precision); +void hll_encode_into_base64(const uint32_t *registers, unsigned char precision, char **blob, size_t *blob_sz); + +#ifdef __cplusplus +} +#endif
\ No newline at end of file diff --git a/src/cells/spread_sketch.c b/src/cells/spread_sketch.c new file mode 100644 index 0000000..85f123b --- /dev/null +++ b/src/cells/spread_sketch.c @@ -0,0 +1,719 @@ +#include <stdbool.h> +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <math.h> +#include <assert.h> + +#include "xxhash/xxhash.h" +#include "uthash.h" + +#include "spread_sketch.h" +#include "hll_common.h" +#include "exdata.h" + +struct entry { + int ref_count; + void *exdata; + bool dying; + char *key; + size_t key_len; + UT_hash_handle hh; +}; + +struct spread_sketch_scheme { + exdata_new_cb new_fn; + exdata_free_cb free_fn; + exdata_merge_cb merge_fn; + exdata_reset_cb reset_fn; + exdata_copy_cb copy_fn; +}; + +struct entry_table { + struct entry *entry; + + struct spread_sketch_scheme scheme; +}; + +struct bucket { + struct entry *content; + uint32_t level; + long long last_update_ms; // linux timestamp since Jan 1st 1970 + uint32_t *sthll_register; +}; + +struct spread_sketch { + int depth; + int width; + long long time_window_ms; + unsigned char precision; + + // shared states of all sthlls + long long reset_idx; + struct timeval reset_time; + + struct spread_sketch_scheme scheme; + + struct bucket *buckets; + struct entry_table *table; + + int level0_cnt; // used to filter out dummy adding +}; + +static void *default_new_fn(void *arg) { + return NULL; +} +static void default_free_fn(void *exdata) { + return; +} +static void default_merge_fn(void *dest, void *src) { + return; +} +static void default_reset_fn(void *exdata) { + return; +} +static void *default_copy_fn(void *exdata) { + return exdata; +} + +struct spread_sketch_scheme DEFAULT_SCHEME = { + .new_fn = default_new_fn, + .free_fn = default_free_fn, + .merge_fn = default_merge_fn, + .reset_fn = default_reset_fn, + .copy_fn = default_copy_fn +}; + +uint32_t cal_true_level(const struct spread_sketch *ss, int bucket_idx, long long t) { + /* + return f(t), the actual level of bucket, which satisfy: + 1. d 2^f(t)/dt is constants + 2. f(t0 + 2T) = 0 + 3. f((t0) = L ) + */ + struct bucket *bucket = &ss->buckets[bucket_idx]; + if (ss->time_window_ms == 0) { + return bucket->level; + } + + unsigned L = bucket->level; + long long T = ss->time_window_ms; + long long t0 = bucket->last_update_ms; + + assert(t >= t0); + if (t - t0 >= 2 * T) { + return 0; + } + if (L <= 1) { + return t - t0 >= T ? 0 : L; + } + + long long tmp_exp = 1 << L; + double a = ((double)(1 - tmp_exp)) / (2.0 * T); + double b = (double)(tmp_exp); + + return (uint32_t)(log2(a * ((double)(t-t0)) + b) + 0.5); +} + +static inline bool key_equal(const char *key1, size_t key1_len, const char *key2, size_t key2_len) { + if (key1_len != key2_len) { + return false; + } + return memcmp(key1, key2, key1_len) == 0; +} +static inline char *key_dup(const char *key, size_t key_len) { + char *ret = malloc(key_len+1); + memcpy(ret, key, key_len); + ret[key_len] = '\0'; + return ret; +} + +struct entry *smart_ptr_table_find(struct entry_table *table, const char *key, size_t key_len) { + struct entry *ret; + HASH_FIND(hh, table->entry, key, key_len, ret); + if (ret == NULL || ret->dying) { + return NULL; + } + return ret; +} + +struct entry *smart_ptr_table_get(struct entry_table *table, const char *key, size_t key_len, void *arg) { + struct entry *ret; + HASH_FIND(hh, table->entry, key, key_len, ret); + + if (ret != NULL) { + ret->dying = false; + ret->ref_count++; + } else { + ret = malloc(sizeof(struct entry)); + ret->dying = false; + ret->ref_count = 1; + ret->key = key_dup(key, key_len); + ret->key_len = key_len; + if (arg == NULL) { + ret->exdata = NULL; + } else { + ret->exdata = table->scheme.new_fn(arg); + } + HASH_ADD_KEYPTR(hh, table->entry, ret->key, ret->key_len, ret); + } + + return ret; +} + +int smart_ptr_table_release(struct entry_table *table, const char *key, size_t key_len) { + struct entry *ret; + HASH_FIND(hh, table->entry, key, key_len, ret); + if (ret == NULL) { + return -1; + } + + ret->ref_count--; + if (ret->ref_count == 0) { + // printf("release %s\n", key); + HASH_DEL(table->entry, ret); + table->scheme.free_fn(ret->exdata); + free(ret->key); + free(ret); + } + + return 0; +} + +void smart_ptr_table_free(struct entry_table *table) { + struct entry *current, *tmp; + HASH_ITER(hh, table->entry, current, tmp) { + HASH_DEL(table->entry, current); + table->scheme.free_fn(current->exdata); + free(current->key); + free(current); + } + free(table); +} + +struct entry_table *smart_ptr_table_new() { + struct entry_table *table = malloc(sizeof(struct entry_table)); + table->entry = NULL; + + table->scheme = DEFAULT_SCHEME; + + return table; +} + +struct spread_sketch *spread_sketch_new(int depth, int width, unsigned char precision, int time_window_ms, struct timeval now) { + struct spread_sketch *pthis = malloc(sizeof(struct spread_sketch)); + + pthis->depth = depth; + pthis->width = width; + pthis->time_window_ms = time_window_ms; + pthis->precision = precision; + + pthis->reset_idx = 0; + pthis->reset_time = now; + + pthis->buckets = calloc(depth * width, sizeof(struct bucket)); + pthis->scheme = DEFAULT_SCHEME; + pthis->table = smart_ptr_table_new(); + pthis->table->scheme = pthis->scheme; + + for (int i = 0; i < depth * width; i++) { + pthis->buckets[i].sthll_register = hll_create_register(precision); + } + + pthis->level0_cnt = pthis->depth * pthis->width; + + return pthis; +} + +void move_registers_forward(struct spread_sketch *ss, const struct timeval *now) { + if (ss->time_window_ms == 0) { + return; + } + + long long reset_reg_count = hll_get_reset_register_count(ss->precision, ss->time_window_ms, *now, &ss->reset_time); + if (reset_reg_count > 0) { + for (int i = 0; i < ss->depth * ss->width; i++) { + hll_reset_registers(ss->buckets[i].sthll_register, ss->precision, ss->reset_idx, reset_reg_count); + } + hll_advance_reset_index(&ss->reset_idx, reset_reg_count, ss->precision); + } +} + +// return 0 if not added, return 1 if added +int spread_sketch_add_hash(struct spread_sketch *ss, const char *key, size_t key_length, uint64_t item_hash, void *arg, struct timeval now) { + uint32_t level = (uint32_t)__builtin_clzll(item_hash) + 1; + long long now_ms = now.tv_sec * 1000 + now.tv_usec / 1000; + + if (item_hash == DUMMY_ITEM_HASH) { + if (ss->level0_cnt == 0) { + return 0; + } + assert(ss->level0_cnt>0); + } + + // https://www.eecs.harvard.edu/~michaelm/postscripts/tr-02-05.pdf + // A technique from the hashing literature is to use two hash functions h1(x) and h2(x) to simulate additional hash functions of the form gi(x) = h1(x) + ih2(x) + // Assuming that the 128-bit xxhash function is perfect, we can view it as a combination of two 64-bit hash functions. + uint64_t hash_x_tmp = XXH3_64bits_withSeed(key, key_length, 171); + uint32_t hash_x1 = (uint32_t) (hash_x_tmp >> 32); + uint32_t hash_x2 = (uint32_t) hash_x_tmp; + + bool in_sketch = false; + move_registers_forward(ss, &now); + for (int i = 0; i < ss->depth; i++) { + uint32_t hash_x = hash_x1 + i * hash_x2; + int bucket_idx = (hash_x % ss->width) + i * ss->width; + struct bucket *bucket = &ss->buckets[bucket_idx]; + + if (bucket->content != NULL && key_equal(bucket->content->key, bucket->content->key_len, key, key_length)) { + bucket->content->dying = false; + bucket->last_update_ms = now_ms; + + if (bucket->level < level) { + if (bucket->level == 0) { + ss->level0_cnt--; + } + bucket->level = level; + } + in_sketch = true; + } else { + uint32_t true_level = bucket->content == NULL ? 0: cal_true_level(ss, bucket_idx, now_ms); + + if (level > true_level) { + if (bucket->level == 0) { + ss->level0_cnt--; + } + + // printf("update key %s to %s, and level %u to %u, in bucket (r,w)=(%d,%u)\n", bucket->content == NULL ? "NULL": bucket->content->key, key, true_level, level, i, hash_x % ss->width); + const struct entry *content_old = bucket->content; + if (content_old != NULL) { + smart_ptr_table_release(ss->table, content_old->key, content_old->key_len); + } + struct entry *content_new = smart_ptr_table_get(ss->table, key, key_length, arg); + bucket->content = content_new; + + bucket->last_update_ms = now_ms; + bucket->level = level; + + in_sketch = true; + } + } + hll_add_hash(bucket->sthll_register, ss->precision, item_hash); + } + + return in_sketch ? 1 : 0; +} + +int spread_sketch_add(struct spread_sketch *ss, const char *key, size_t key_length, const char* item, size_t item_len, void *arg, struct timeval now) { + uint64_t hash = XXH3_64bits(item, item_len); + return spread_sketch_add_hash(ss, key, key_length, hash, arg, now); +} + +double spread_sketch_point_query(const struct spread_sketch *ss, const char *key, size_t key_length) { + uint64_t hash_x_tmp = XXH3_64bits_withSeed(key, key_length, 171); + uint32_t hash_x1 = (uint32_t) (hash_x_tmp >> 32); + uint32_t hash_x2 = (uint32_t) hash_x_tmp; + + double count_min = (double)INT32_MAX; + for (int i = 0; i < ss->depth; i++) { + uint32_t hash_x = hash_x1 + i * hash_x2; + int bucket_idx = (hash_x % ss->width) + i * ss->width; + + double est = hll_count(ss->buckets[bucket_idx].sthll_register, ss->precision, ss->reset_idx, ss->time_window_ms); + if (est < count_min) { + count_min = est; + } + } + return count_min; +} + +struct spread_sketch *duplicate_and_step(const struct spread_sketch *ss, const struct timeval *now) { + struct spread_sketch *duplicate = malloc(sizeof(struct spread_sketch)); + duplicate->depth = ss->depth; + duplicate->width = ss->width; + duplicate->precision = ss->precision; + duplicate->reset_idx = ss->reset_idx; + duplicate->reset_time = ss->reset_time; + duplicate->time_window_ms = ss->time_window_ms; + duplicate->buckets = calloc(ss->depth * ss->width, sizeof(struct bucket)); + for (int i = 0; i < ss->depth * ss->width; i++) { + duplicate->buckets[i].sthll_register = hll_duplicate(ss->buckets[i].sthll_register, ss->precision); + } + + move_registers_forward(duplicate, now); + return duplicate; +} + +double spread_sketch_query(const struct spread_sketch *ss, const char *key, size_t key_length, struct timeval now) { + if (ss->time_window_ms!=0 && hll_should_slide(ss->precision, ss->time_window_ms, now, ss->reset_time)) { + struct spread_sketch *duplicate = duplicate_and_step(ss, &now); + double ret = spread_sketch_point_query(duplicate, key, key_length); + spread_sketch_free(duplicate); + return ret; + } else { + return spread_sketch_point_query(ss, key, key_length); + } +} + +void spread_sketch_free(struct spread_sketch *ss) { + if (ss == NULL) { + return; + } + smart_ptr_table_free(ss->table); + for (int i = 0; i < ss->depth * ss->width; i++) { + hll_free_register(ss->buckets[i].sthll_register); + } + free(ss->buckets); + free(ss); +} + +void spread_sketch_merge(struct spread_sketch *dst, const struct spread_sketch *src) +{ + assert(dst->depth == src->depth && dst->width == src->width); + assert(dst->time_window_ms == src->time_window_ms); + assert(dst->precision == src->precision); + + for (int i = 0; i < dst->depth * dst->width; i++) { + const struct bucket *bucket_src = &src->buckets[i]; + struct bucket *bucket_dst = &dst->buckets[i]; + + if (bucket_src->content == NULL || bucket_src->content->dying) { + continue; + } + + hll_merge(bucket_dst->sthll_register, src->buckets[i].sthll_register, dst->precision); + if (bucket_dst->level == 0 && bucket_src->level > 0) { + dst->level0_cnt--; + } + + if (bucket_dst->content == NULL) { + bucket_dst->content = smart_ptr_table_get(dst->table, bucket_src->content->key, bucket_src->content->key_len, NULL); + bucket_dst->level = bucket_src->level; + bucket_dst->last_update_ms = bucket_src->last_update_ms; + + continue; + } + bucket_dst->content->dying = false; + + if (key_equal(bucket_src->content->key, bucket_src->content->key_len, bucket_dst->content->key, bucket_dst->content->key_len)) { + if (bucket_src->level > bucket_dst->level) { + bucket_dst->level = bucket_src->level; + } + if (bucket_src->last_update_ms > bucket_dst->last_update_ms) { + bucket_dst->last_update_ms = bucket_src->last_update_ms; + } + } else { + uint32_t true_level_src = cal_true_level(src, i, bucket_src->last_update_ms); + uint32_t true_level_dst = cal_true_level(dst, i, bucket_dst->last_update_ms); + + if (true_level_src > true_level_dst) { + smart_ptr_table_release(dst->table, bucket_dst->content->key, bucket_dst->content->key_len); + bucket_dst->content = smart_ptr_table_get(dst->table, bucket_src->content->key, bucket_src->content->key_len, NULL); + bucket_dst->last_update_ms = bucket_src->last_update_ms; + } + } + } + + // for exdata + const struct spread_sketch_scheme *scheme = &dst->table->scheme; + + struct entry *content_dest, *content_src, *tmp; + HASH_ITER(hh, dst->table->entry, content_dest, tmp) { + HASH_FIND(hh, src->table->entry, content_dest->key, content_dest->key_len, content_src); + if (content_src == NULL || content_src->dying) { + continue; + } + + if (content_dest->exdata == NULL) { + content_dest->exdata = scheme->copy_fn(content_src->exdata); + } else { + scheme->merge_fn(content_dest->exdata, content_src->exdata); + } + } +} + +void *spread_sketch_get0_exdata(const struct spread_sketch *ss, const char *key, size_t key_len) { + const struct entry *content = smart_ptr_table_find(ss->table, key, key_len); + if (content == NULL) { + return NULL; + } + return content->exdata; +} + +void spread_sketch_reset(struct spread_sketch *ss) { + for (int i = 0; i < ss->depth * ss->width; i++) { + struct bucket *bucket = &ss->buckets[i]; + bucket->level = 0; + hll_reset_registers(bucket->sthll_register, ss->precision, ss->reset_idx, 100000); // count is big enough + } + + struct entry *content, *tmp; + HASH_ITER(hh, ss->table->entry, content, tmp) { + ss->scheme.reset_fn(content->exdata); + content->dying = true; + } + + ss->level0_cnt = ss->depth * ss->width; +} + +void spread_sketch_set_exdata_schema(struct spread_sketch *ss, exdata_new_cb new_fn, exdata_free_cb free_fn, exdata_merge_cb merge_fn, exdata_reset_cb reset_fn, exdata_copy_cb copy_fn) { + ss->scheme.new_fn = new_fn; + ss->scheme.free_fn = free_fn; + ss->scheme.merge_fn = merge_fn; + ss->scheme.reset_fn = reset_fn; + ss->scheme.copy_fn = copy_fn; + + ss->table->scheme = ss->scheme; +} + +int spread_sketch_get_count(const struct spread_sketch *ss) { + int cnt = 0; + struct entry *content, *tmp; + HASH_ITER(hh, ss->table->entry, content, tmp) { + if (!content->dying) { + cnt++; + } + } + + return cnt; +} + +size_t spread_sketch_list(const struct spread_sketch *ss, void **exdatas, size_t n_exdatas) { + size_t count = 0; + struct entry *content, *tmp; + HASH_ITER(hh, ss->table->entry, content, tmp) { + if (content->dying) { + continue; + } + if (count >= n_exdatas) { + break; + } + exdatas[count] = content->exdata; + count++; + } + return count; +} + +void spread_sketch_list_keys(const struct spread_sketch *ss, char ***keys, size_t **key_lens, size_t *n_keys) { + size_t count_max = HASH_COUNT(ss->table->entry); + size_t count = 0; + + *keys = malloc(count_max * sizeof(char *)); + *key_lens = malloc(count_max * sizeof(size_t)); + + struct entry *content, *tmp; + HASH_ITER(hh, ss->table->entry, content, tmp) { + if (content->dying) { + continue; + } + + (*keys)[count] = content->key; + (*key_lens)[count] = content->key_len; + count++; + } + + *n_keys = count; +} + +double spread_sketch_get_cardinality(const struct spread_sketch *ss, const char *key, size_t key_len) { + if (smart_ptr_table_find(ss->table, key, key_len) == NULL) { + return -1; + } + + double est = spread_sketch_point_query(ss, key, key_len); + return est; +} + +char *spread_sketch_get_hll_base64_serialization(const struct spread_sketch *ss, const char *key, size_t key_len) { + uint64_t hash_x_tmp = XXH3_64bits_withSeed(key, key_len, 171); + uint32_t hash_x1 = (uint32_t) (hash_x_tmp >> 32); + uint32_t hash_x2 = (uint32_t) hash_x_tmp; + uint32_t *register_ret = NULL; + double min_cardinality = (double)INT32_MAX; + + for (int i = 0; i < ss->depth; i++) { + uint32_t hash_x = hash_x1 + i * hash_x2; + int bucket_idx = (hash_x % ss->width) + i * ss->width; + + double est = hll_count(ss->buckets[bucket_idx].sthll_register, ss->precision, ss->reset_idx, ss->time_window_ms); + if (est < min_cardinality) { + min_cardinality = est; + register_ret = ss->buckets[bucket_idx].sthll_register; + } + } + + char *enc; + size_t enc_len; + hll_encode_into_base64(register_ret, ss->precision, &enc, &enc_len); + return enc; +} + +struct spread_sketch *spread_sketch_copy(const struct spread_sketch *src) { + struct spread_sketch *dst = malloc(sizeof(struct spread_sketch)); + memcpy(dst, src, sizeof(struct spread_sketch)); + + dst->buckets = calloc(dst->depth * dst->width, sizeof(struct bucket)); + dst->table = smart_ptr_table_new(); + spread_sketch_set_exdata_schema(dst, src->scheme.new_fn, src->scheme.free_fn, src->scheme.merge_fn, src->scheme.reset_fn, src->scheme.copy_fn); + for (int i = 0; i < dst->depth * dst->width; i++) { + dst->buckets[i].sthll_register = hll_duplicate(src->buckets[i].sthll_register, src->precision); + } + + for (int i = 0; i < dst->depth * dst->width; i++) { + if (src->buckets[i].content == NULL || src->buckets[i].content->dying) { + continue; + } + dst->buckets[i].content = smart_ptr_table_get(dst->table, src->buckets[i].content->key, src->buckets[i].content->key_len, NULL); + dst->buckets[i].level = src->buckets[i].level; + if (dst->buckets[i].content->exdata == NULL) { + dst->buckets[i].content->exdata = src->scheme.copy_fn(src->buckets[i].content->exdata); + } + } + return dst; +} + +void spread_sketch_get_parameter_recommendation(int expected_super_spreader_number, int *depth_out, int *width_out, unsigned char *precision_out) +{ + int logk = expected_super_spreader_number >= 3200 ? 4 : 3; // lg3200 = 3.51,round up to 4 + *depth_out = logk; + + int w; + if (expected_super_spreader_number <= 100) { + w = expected_super_spreader_number * 3 /2; // * 1.5, when the number is small, we need more width + } else { + w = expected_super_spreader_number + 50; // + 50: 100*1.5-100 = 50, make w=f(k) continuous + } + if (w < 40) { + w = 40; + } + *width_out = w; + + *precision_out = 6; +} + +void spread_sketch_get_parameter(const struct spread_sketch *ss, int *depth_out, int *width_out, unsigned char *precision_out, int *time_window_ms_out) +{ + *depth_out = ss->depth; + *width_out = ss->width; + *precision_out = ss->precision; + *time_window_ms_out = ss->time_window_ms; +} + +void spread_sketch_change_precision(struct spread_sketch *ss, unsigned char precision) { + for (int i = 0; i < ss->depth * ss->width; i++) { + hll_free_register(ss->buckets[i].sthll_register); + ss->buckets[i].sthll_register = hll_create_register(precision); + } + ss->precision = precision; +} + +void spread_sketch_serialize(const struct spread_sketch *ss, char **blob, size_t *blob_sz) +{ + /* + format: + struct spread_sketch(including useless pointers) + struct bucket * depth * width + int64_t * depth * width for keylen + key（close-knit, key_len, key, key_len, key, ..., in order of bucket index） + */ + // get serialize size + + size_t sz = 0; + sz += sizeof(struct spread_sketch); + sz += ss->depth * ss->width * (sizeof(struct bucket) + hll_size(ss->precision) + sizeof(int64_t)); + for (int i = 0; i < ss->depth * ss->width; i++) { + if (ss->buckets[i].content != NULL) { + sz += ss->buckets[i].content->key_len; + } + } + + char *buffer = malloc(sz); + *blob = buffer; + *blob_sz = sz; + + memcpy(buffer, ss, sizeof(struct spread_sketch)); + buffer += sizeof(struct spread_sketch); + for (int i = 0; i < ss->depth * ss->width; i++) { + const struct bucket *bucket = &ss->buckets[i]; + memcpy(buffer, bucket, sizeof(struct bucket)); + buffer += sizeof(struct bucket); + memcpy(buffer, bucket->sthll_register, hll_size(ss->precision)); + buffer += hll_size(ss->precision); + } + for (int i = 0; i < ss->depth * ss->width; i++) { + int64_t key_len; + if (ss->buckets[i].content != NULL) { + key_len = ss->buckets[i].content->key_len; + } else { + key_len = 0; + } + memcpy(buffer, &key_len, sizeof(int64_t)); + buffer += sizeof(int64_t); + + if (key_len > 0) { + memcpy(buffer, ss->buckets[i].content->key, key_len); + buffer += key_len; + } + } +} + +struct spread_sketch *spread_sketch_deserialize(const char *blob, size_t blob_sz) +{ + struct spread_sketch *ss = malloc(sizeof(struct spread_sketch)); + memcpy(ss, blob, sizeof(struct spread_sketch)); + blob += sizeof(struct spread_sketch); + ss->table = smart_ptr_table_new(); + ss->buckets = calloc(ss->depth * ss->width, sizeof(struct bucket)); + ss->scheme = DEFAULT_SCHEME; + + for (int i = 0; i < ss->depth * ss->width; i++) { + struct bucket *bucket = &ss->buckets[i]; + memcpy(bucket, blob, sizeof(struct bucket)); + blob += sizeof(struct bucket); + + bucket->sthll_register = hll_create_register(ss->precision); + memcpy(bucket->sthll_register, blob, hll_size(ss->precision)); + blob += hll_size(ss->precision); + } + + for (int i = 0; i < ss->depth * ss->width; i++) { + int64_t key_len; + memcpy(&key_len, blob, sizeof(int64_t)); + blob += sizeof(int64_t); + if (key_len == 0) { + continue; + } + const char *key = blob; + blob += key_len; + + struct entry *content = smart_ptr_table_get(ss->table, key, key_len, NULL); + ss->buckets[i].content = content; + } + + return ss; +} + +void spread_sketch_merge_blob(struct spread_sketch *dst, const char *blob, size_t blob_sz) +{ + struct spread_sketch *src = spread_sketch_deserialize(blob, blob_sz); + spread_sketch_merge(dst, src); + spread_sketch_free(src); +} + +size_t spread_sketch_calculate_memory_usage(const struct spread_sketch *ss) +{ + size_t ret = 0; + ret += sizeof(struct spread_sketch); + + size_t bucket_size = sizeof(struct bucket) + hll_size(ss->precision); + printf("every bucket size: %zu\n", bucket_size); + ret += ss->depth * ss->width * bucket_size; + + printf("the number of content: %u\n", HASH_COUNT(ss->table->entry)); + struct entry *content, *tmp; + HASH_ITER(hh, ss->table->entry, content, tmp) { + ret += sizeof(struct entry); + ret += content->key_len; + // assume the exdata is NULL + } + return ret; +}
\ No newline at end of file diff --git a/src/cells/spread_sketch.h b/src/cells/spread_sketch.h new file mode 100644 index 0000000..c15ad81 --- /dev/null +++ b/src/cells/spread_sketch.h @@ -0,0 +1,55 @@ +#pragma once + + + +#ifdef __cplusplus +extern "C"{ +#endif + +#include "exdata.h" +#include <stdint.h> //uint64_t +#include <sys/time.h> // struct timeval +#include <stddef.h> + +#define DUMMY_ITEM_HASH (1ULL<<63) // level(left most zeros) = 0 + +struct spread_sketch; + +// set time_window_ms to zero, the spread sketch will not slide and decay, in which case, `now` can be any value +struct spread_sketch *spread_sketch_new(int depth, int width, unsigned char precision, int time_window_ms, struct timeval now); +void spread_sketch_free(struct spread_sketch *ss); +void spread_sketch_set_exdata_schema(struct spread_sketch *ss, exdata_new_cb new_fn, exdata_free_cb free_fn, exdata_merge_cb merge_fn, exdata_reset_cb reset_fn, exdata_copy_cb copy_fn); + +int spread_sketch_add_hash(struct spread_sketch *ss, const char *key, size_t key_length, uint64_t item_hash, void *arg, struct timeval now); +int spread_sketch_add(struct spread_sketch *ss, const char *key, size_t key_length, const char* item, size_t item_len, void *arg, struct timeval now); + +// get the number of keys stored in spread sketch +int spread_sketch_get_count(const struct spread_sketch *ss); +// list all the keys in spread sketch. User should free the arrays, but do not free the elements of strings in the array(because they are references to the internal data structure) +// Example: char **key; size_t *key_len; size_t n_keys; spread_sketch_list_keys(&key, &key_len, &n_keys); free(key); free(key_len); +void spread_sketch_list_keys(const struct spread_sketch *ss, char ***keys, size_t **key_lens, size_t *n_keys); +// query the cardinality(or fanout) of a key in spread sketch. +// Even thought spread sketch algorithm does not requires keys to exist innately, when querying a key that is not present in the spread sketch, `spread_sketch_get_cardinality` will return -1. +double spread_sketch_get_cardinality(const struct spread_sketch *ss, const char *key, size_t key_len); +// query a hyperloglog 's base64 serialization. The serialization format is [1,precision,register...] and then encoded by base64 +char *spread_sketch_get_hll_base64_serialization(const struct spread_sketch *ss, const char *key, size_t key_len); +void *spread_sketch_get0_exdata(const struct spread_sketch *ss, const char *key, size_t key_len); +// in most cases, it has the same output as `spread_sketch_get_cardinality`, but it will perform more like an ordinary spread sketch query. +// Will always return a value, even if the key is not present in the spread sketch. Must pass a `now` value required by Stagger hll query. +double spread_sketch_query(const struct spread_sketch *ss, const char *key, size_t key_length, struct timeval now); + +void spread_sketch_merge(struct spread_sketch *dest, const struct spread_sketch *src); +struct spread_sketch *spread_sketch_copy(const struct spread_sketch *src); +void spread_sketch_serialize(const struct spread_sketch *ss, char **blob, size_t *blob_sz); +struct spread_sketch *spread_sketch_deserialize(const char *blob, size_t blob_sz); +void spread_sketch_merge_blob(struct spread_sketch *dst, const char *blob, size_t blob_sz); +void spread_sketch_reset(struct spread_sketch *ss); + +void spread_sketch_get_parameter(const struct spread_sketch *ss, int *depth_out, int *width_out, unsigned char *precision_out, int *time_window_ms_out); +// spread sketch alway store values more than expected_query_num,expected_query_num is a hint to set spread sketch parameters properly +void spread_sketch_get_parameter_recommendation(int expected_super_spreader_number, int *depth_out, int *width_out, unsigned char *precision_out); +size_t spread_sketch_calculate_memory_usage(const struct spread_sketch *ss); + +#ifdef __cplusplus +} +#endif
\ No newline at end of file diff --git a/src/cube.c b/src/cube.c new file mode 100644 index 0000000..9dc7ab3 --- /dev/null +++ b/src/cube.c @@ -0,0 +1,1393 @@ + +#include <stdio.h> +#include <stdlib.h> +#include <assert.h> +#include <string.h> +#include <stdarg.h> + +#include "uthash.h" +#define XXH_INLINE_ALL +#include "xxhash/xxhash.h" + +#include "cube.h" +#include "metric_manifest.h" +#include "metric.h" +#include "heavy_keeper.h" +#include "hash_table.h" +#include "spread_sketch.h" + +#define DEFAULT_N_METRIC 32 +#define DEFAULT_N_CUBE 64 +static const struct timeval DUMMY_TIME_VAL = {0, 0}; + +struct exdata_new_args { + const struct field *cell_dimensions; + size_t n_dimensions; +}; + +struct cube_manager { + struct cube *hash_table; // the key of cube is serialized cube dimensions + + struct cube **cube_slots; + size_t next_index; // next_index + size_t slots_number; +}; + +struct cell { + struct metric **slots; + size_t next_index; //index of next available slot + size_t slots_number; + struct field_list cell_dimensions; +}; + +struct cube { + enum sampling_mode sampling_mode; + union { + struct heavy_keeper *heavykeeper; + struct hash_table *table; + struct spread_sketch *spread_sketch; + }; + size_t max_n_cell; + struct metric_manifest_manager *manifest_manager; + + struct field *cube_dimensions; + size_t n_dimensions; + + int primary_metric_id; + char *serialized_dimensions; // the key of cube is serialized cube dimensions + size_t serialized_dimensions_len; + int id; + UT_hash_handle hh; +}; + +static struct field *field_array_duplicate(const struct field *fields_src, size_t n_field) +{ + struct field *ret = malloc(sizeof(struct field) * n_field); + + for (size_t i = 0; i < n_field; i++) { + ret[i].key = strdup(fields_src[i].key); + ret[i].type = fields_src[i].type; + switch (fields_src[i].type) + { + case FIELD_VALUE_INTEGER: + ret[i].value_longlong = fields_src[i].value_longlong; + break; + case FIELD_VALUE_CSTRING: + ret[i].value_str = strdup(fields_src[i].value_str); + break; + case FIELD_VALUE_DOUBLE: + ret[i].value_double = fields_src[i].value_double; + break; + default: + break; + } + } + + return ret; +} + +void add_cube_to_position(struct cube_manager *pthis, struct cube *cube, int id) +{ + if (id >= pthis->slots_number) { + struct cube **old_cube_arr = pthis->cube_slots; + pthis->cube_slots = calloc(pthis->slots_number * 2, sizeof(struct cube *)); + memcpy(pthis->cube_slots, old_cube_arr, sizeof(struct cube *) * pthis->slots_number); + free(old_cube_arr); + + pthis->slots_number *= 2; + } + pthis->cube_slots[id] = cube; + + if (id >= pthis->next_index) { + pthis->next_index = id + 1; + } +} + +void cube_manager_free(struct cube_manager *pthis) { + struct cube *node = NULL; + struct cube *tmp = NULL; + struct cube *head = pthis->hash_table; + HASH_ITER(hh, head, node, tmp) { + HASH_DEL(head, node); + cube_free(node); + } + + free(pthis->cube_slots); + free(pthis); +} + +struct cube_manager *cube_manager_new() { + struct cube_manager *pthis = (struct cube_manager *)malloc(sizeof(struct cube_manager)); + pthis->hash_table = NULL; + + pthis->cube_slots = (struct cube **)calloc(DEFAULT_N_CUBE, sizeof(struct cube *)); + pthis->next_index = 0; + pthis->slots_number = DEFAULT_N_CUBE; + return pthis; +} + +void print_field_array(const struct field *fields, size_t n_field) +{ + printf("dimension with length %zu \n", n_field); + for (size_t i = 0; i < n_field; i++) { + printf("%s: ", fields[i].key); + switch (fields[i].type) + { + case FIELD_VALUE_INTEGER: + printf("%lld\n", fields[i].value_longlong); + break; + case FIELD_VALUE_DOUBLE: + printf("%lf\n", fields[i].value_double); + break; + case FIELD_VALUE_CSTRING: + printf("%s\n", fields[i].value_str); + break; + default: + break; + } + } +} + +static void field_array_to_key(const struct field fields[], size_t n_fields, char **out_key, size_t *out_key_size) +{ + if (n_fields == 0) { + // use a default dummy key + *out_key = strdup("\a\tN"); + *out_key_size = strlen(*out_key); + return; + } + int i = 0; + size_t used_len = 0; + struct field *field = NULL; + + size_t alloced_every_time = 1024; + size_t remain_key_size = 1024; + size_t total_key_size = 1024; + char *dynamic_mem = (char *)malloc(total_key_size); + void *val_position = NULL; + + size_t key_len = 0; + size_t val_len = 0; + + for(i = 0; i < (int)n_fields; i++) + { + field = (struct field *)&fields[i]; + key_len = strlen(field->key); + switch(field->type) + { + case FIELD_VALUE_INTEGER: + val_len = sizeof(long long); + val_position = (void *)&field->value_longlong; + break; + case FIELD_VALUE_DOUBLE: + val_len = sizeof(double); + val_position = (void *)&field->value_double; + + break; + case FIELD_VALUE_CSTRING: + val_len = strlen(field->value_str); + val_position = (void *)field->value_str; + break; + default: + assert(0); + break; + } + + used_len = key_len + val_len; + while (used_len >= remain_key_size) { + total_key_size += alloced_every_time; + remain_key_size += alloced_every_time; + dynamic_mem = (char *)realloc(dynamic_mem, total_key_size); + } + memcpy(dynamic_mem + total_key_size - remain_key_size, field->key, key_len); + memcpy(dynamic_mem + total_key_size - remain_key_size + key_len, val_position, val_len); + remain_key_size -= used_len; + } + + *out_key = dynamic_mem; + *out_key_size = total_key_size - remain_key_size; +} + +int cube_manager_add(struct cube_manager *pthis, struct cube *cube) +{ + char *key = cube->serialized_dimensions; + size_t key_len = cube->serialized_dimensions_len; + + struct cube *old_cube = NULL; + HASH_FIND(hh, pthis->hash_table, key, key_len, old_cube); + if (old_cube != NULL) { + return -1; + } + + int id = 0; + for ( ;id < pthis->next_index; id++) { + if (pthis->cube_slots[id] == NULL) { + break; + } + } + + cube->id = id; + HASH_ADD_KEYPTR(hh, pthis->hash_table, cube->serialized_dimensions, key_len, cube); + + add_cube_to_position(pthis, cube, id); + + return id; +} + +void cube_manager_delete(struct cube_manager *pthis, struct cube *cube) +{ + int id = cube->id; + HASH_DEL(pthis->hash_table, cube); + + cube_free(cube); + pthis->cube_slots[id] = NULL; + if (id == pthis->next_index - 1) { + pthis->next_index--; + } +} + +int cube_manager_find(const struct cube_manager *pthis, const struct field *cube_dimensions, size_t n_dimension) +{ + char *key; + size_t key_len; + field_array_to_key(cube_dimensions, n_dimension, &key, &key_len); + + struct cube *node = NULL; + HASH_FIND(hh, pthis->hash_table, key, key_len, node); + + free(key); + if (node == NULL) { + return -1; + } else { + return node->id; + } +} + +struct cube *cube_manager_get_cube_by_id(const struct cube_manager *manager, int cube_id) { + if (cube_id < 0 || cube_id >= manager->slots_number) { + return NULL; + } + return manager->cube_slots[cube_id]; +} + +void cube_manager_list(const struct cube_manager *pthis, int **cube_ids, int *n_cube) { + int all_available_cube_count = 0; + + int *tmp_ids = (int *)malloc(sizeof(int) * pthis->next_index); + for (int i = 0; i < pthis->next_index; i++) { + if (pthis->cube_slots[i] != NULL) { + tmp_ids[all_available_cube_count++] = i; + } + } + if (all_available_cube_count == 0) { + free(tmp_ids); + *cube_ids = NULL; + *n_cube = 0; + return; + } + + *cube_ids = tmp_ids; + *n_cube = all_available_cube_count; +} + +void cube_manager_calibrate(struct cube_manager *pthis, const struct cube_manager *master) +{ + struct cube *node_in_master, *node_in_dest, *tmp; + + HASH_ITER(hh, pthis->hash_table, node_in_dest, tmp) { + HASH_FIND(hh, master->hash_table, node_in_dest->serialized_dimensions, node_in_dest->serialized_dimensions_len, node_in_master); + + if (node_in_master == NULL) { // exist in self but not in master + cube_manager_delete(pthis, node_in_dest); + } else { + metric_manifest_manager_free(node_in_dest->manifest_manager); + node_in_dest->manifest_manager = metric_manifest_manager_copy(node_in_master->manifest_manager); + } + } + + // exist in master but not in self + HASH_ITER(hh, master->hash_table, node_in_master, tmp) { + HASH_FIND(hh, pthis->hash_table, node_in_master->serialized_dimensions, node_in_master->serialized_dimensions_len, node_in_dest); + + if (node_in_dest == NULL) { + cube_manager_add(pthis, cube_fork(node_in_master)); + } + } +} + +struct cube_manager *cube_manager_fork(const struct cube_manager *src) +{ + struct cube_manager *pthis = cube_manager_new(); + struct cube *node = NULL; + struct cube *tmp = NULL; + HASH_ITER(hh, src->hash_table, node, tmp) { + cube_manager_add(pthis, cube_fork(node)); + } + return pthis; +} + +int cube_manager_merge(struct cube_manager *dest, const struct cube_manager *src) +{ + struct cube *node = NULL; + struct cube *tmp = NULL; + int ret = FS_OK; + HASH_ITER(hh, src->hash_table, node, tmp) { + struct cube *node_in_dest = NULL; + HASH_FIND(hh, dest->hash_table, node->serialized_dimensions, node->serialized_dimensions_len, node_in_dest); + + if (node_in_dest == NULL) { + cube_manager_add(dest, cube_copy(node)); + } else { + int tmp_ret = cube_merge(node_in_dest, node); + if (tmp_ret != FS_OK) { + ret = tmp_ret; + } + } + } + + return ret; +} + +void cube_manager_reset(struct cube_manager *pthis) +{ + for (int i = 0; i < pthis->next_index; i++) { + if (pthis->cube_slots[i] == NULL) { + continue; + } + cube_reset(pthis->cube_slots[i]); + } +} + +struct metric *find_metric_in_cell(const struct cell *cell, int metric_id) +{ + if (metric_id >= cell->next_index) { + return NULL; + } + return cell->slots[metric_id]; +} + +void add_metric_to_cell(struct cell *cell, struct metric *metric, int metric_id) +{ + if (metric_id >= cell->slots_number) { + cell->slots = realloc(cell->slots, sizeof(struct metric *) * cell->slots_number * 2); + memset(cell->slots + cell->slots_number, 0, sizeof(struct metric *) * cell->slots_number); + cell->slots_number *= 2; + } + + cell->slots[metric_id] = metric; + + if (metric_id >= cell->next_index) { + cell->next_index = metric_id + 1; + } +} + +struct metric *add_or_find_metric_in_cell(const struct metric_manifest *manifest, struct cell *cell) +{ + struct metric *metric = find_metric_in_cell(cell, manifest->id); + if (metric != NULL) { + return metric; + } + + metric = metric_new(manifest); + add_metric_to_cell(cell, metric, manifest->id); + return metric; +} + +struct cell *cell_new(const struct exdata_new_args *args) { + struct cell *pthis = malloc(sizeof(struct cell)); + pthis->slots = calloc(DEFAULT_N_METRIC, sizeof(struct metric *)); + pthis->slots_number = DEFAULT_N_METRIC; + pthis->next_index = 0; + + pthis->cell_dimensions.n_field = args->n_dimensions; + pthis->cell_dimensions.field = field_array_duplicate(args->cell_dimensions, args->n_dimensions); + return pthis; +} + +void cell_free(struct cell *pthis) { + for (size_t i = 0; i < pthis->next_index; i++) { + metric_free(pthis->slots[i]); + } + free(pthis->slots); + for (size_t i = 0; i < pthis->cell_dimensions.n_field; i++) { + free((char *)pthis->cell_dimensions.field[i].key); + if (pthis->cell_dimensions.field[i].type == FIELD_VALUE_CSTRING) { + free((char *)pthis->cell_dimensions.field[i].value_str); + } + } + free(pthis->cell_dimensions.field); + free(pthis); +} + +struct cell *cell_copy(const struct cell *src) { + struct cell *pthis = malloc(sizeof(struct cell)); + pthis->slots = calloc(src->slots_number, sizeof(struct metric *)); + pthis->slots_number = src->slots_number; + pthis->next_index = src->next_index; + for (size_t i = 0; i < src->next_index; i++) { + if (src->slots[i] == NULL) { + continue; + } + + pthis->slots[i] = metric_copy(src->slots[i]); + } + + pthis->cell_dimensions.n_field = src->cell_dimensions.n_field; + pthis->cell_dimensions.field = field_array_duplicate(src->cell_dimensions.field, src->cell_dimensions.n_field); + + return pthis; +} + +void cell_reset(struct cell *pthis) { + for (size_t i = 0; i < pthis->next_index; i++) { + if (pthis->slots[i] == NULL) { + continue; + } + metric_reset(pthis->slots[i]); + } +} + +void cell_merge(struct cell *dest, const struct cell *src) { + for (size_t i = 0; i < src->next_index; i++) { + const struct metric *metric_src = src->slots[i]; + if (metric_src == NULL) { + continue; + } + struct metric *metric_dst = find_metric_in_cell(dest, i); + + if (metric_dst == NULL) { + metric_dst = metric_copy(metric_src); + add_metric_to_cell(dest, metric_dst, i); + } else { + metric_merge(metric_dst, metric_src); + } + } +} + +void *exdata_new_i(void *arg) { + return cell_new((struct exdata_new_args *)arg); +} + +void exdata_free_i(void *exdata) { + cell_free((struct cell *)exdata); +} + +void exdata_reset_i(void *exdata) { + cell_reset((struct cell *)exdata); +} + +void exdata_merge_i(void *dest, void *src) { + cell_merge((struct cell *)dest, (struct cell *)src); +} + +void *exdata_copy_i(void *exdata) { + return cell_copy((struct cell *)exdata); +} + +struct cube *cube_info_new(const struct field *dimensions, size_t n_dimensions) +{ + struct cube *cube = calloc(1, sizeof(struct cube)); + + if (n_dimensions == 0) { + cube->cube_dimensions = NULL; + } else { + cube->cube_dimensions = field_array_duplicate(dimensions, n_dimensions); + } + + cube->n_dimensions = n_dimensions; + field_array_to_key(dimensions, n_dimensions, &cube->serialized_dimensions, &cube->serialized_dimensions_len); + + cube->id = -1; + cube->primary_metric_id = -1; + + return cube; +} + +struct cube *cube_new(const struct field *dimensions, size_t n_dimensions) +{ + struct cube *cube = cube_info_new(dimensions, n_dimensions); + cube->manifest_manager = metric_manifest_manager_new(); + + return cube; +} + +int cube_set_sampling(struct cube *cube, enum sampling_mode mode, int max_n_cell, int primary_metric_id) { + if (cube->sampling_mode == mode && cube->max_n_cell == max_n_cell && cube->primary_metric_id == primary_metric_id) { + return FS_OK; + } + const struct metric_manifest *manifest = metric_manifest_manager_get_by_id(cube->manifest_manager, primary_metric_id); + if (manifest == NULL && mode != SAMPLING_MODE_COMPREHENSIVE) { + return FS_ERR_INVALID_METRIC_ID; + } + if ((mode == SAMPLING_MODE_TOPK && manifest->type != METRIC_TYPE_COUNTER) || + (mode == SAMPLING_MODE_TOP_CARDINALITY && manifest->type != METRIC_TYPE_HLL)) { + return FS_ERR_INVALID_METRIC_TYPE; + } + + if (cube->primary_metric_id != -1) { + // delete previous settings + switch (cube->sampling_mode) + { + case SAMPLING_MODE_TOPK: + heavy_keeper_free(cube->heavykeeper); + break; + case SAMPLING_MODE_COMPREHENSIVE: + hash_table_free(cube->table); + break; + case SAMPLING_MODE_TOP_CARDINALITY: + spread_sketch_free(cube->spread_sketch); + break; + default: + assert(0); + break; + } + } + + switch (mode) + { + case SAMPLING_MODE_TOPK: + cube->heavykeeper = heavy_keeper_new(max_n_cell); + heavy_keeper_set_exdata_schema(cube->heavykeeper, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i); + break; + case SAMPLING_MODE_COMPREHENSIVE: + cube->table = hash_table_new(max_n_cell); + hash_table_set_exdata_schema(cube->table, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i); + break; + case SAMPLING_MODE_TOP_CARDINALITY: { + int width, depth; + unsigned char precision; + spread_sketch_get_parameter_recommendation(max_n_cell, &depth, &width, &precision); + cube->spread_sketch = spread_sketch_new(depth, width, precision, 0, DUMMY_TIME_VAL); + spread_sketch_set_exdata_schema(cube->spread_sketch, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i); + break; } + default: + assert(0); + break; + } + + cube->sampling_mode = mode; + cube->max_n_cell = max_n_cell; + cube->primary_metric_id = primary_metric_id; + + return FS_OK; +} + +void cube_free(struct cube *cube) { + switch (cube->sampling_mode) + { + case SAMPLING_MODE_TOPK: + heavy_keeper_free(cube->heavykeeper); + break; + case SAMPLING_MODE_COMPREHENSIVE: + hash_table_free(cube->table); + break; + case SAMPLING_MODE_TOP_CARDINALITY: + spread_sketch_free(cube->spread_sketch); + break; + default: + assert(0); + break; + } + + for (size_t i = 0; i < cube->n_dimensions; i++) { + struct field *field = &cube->cube_dimensions[i]; + free((char *)field->key); + if (field->type == FIELD_VALUE_CSTRING) { + free((char *)field->value_str); + } + } + free(cube->cube_dimensions); + + free(cube->serialized_dimensions); + metric_manifest_manager_free(cube->manifest_manager); + + free(cube); +} + +void cube_reset(struct cube *cube) { + switch (cube->sampling_mode) + { + case SAMPLING_MODE_TOPK: + heavy_keeper_reset(cube->heavykeeper); + break; + case SAMPLING_MODE_COMPREHENSIVE: + hash_table_reset(cube->table); + break; + case SAMPLING_MODE_TOP_CARDINALITY: + spread_sketch_reset(cube->spread_sketch); + break; + default: + assert(0); + break; + } +} + +struct cell *get_cell_in_cube_generic(struct cube *cube, const struct field *dimensions, size_t n_dimensions) { + char *compound_dimension; + size_t compound_dimension_len; + field_array_to_key(dimensions, n_dimensions, &compound_dimension, &compound_dimension_len); + + struct exdata_new_args args; + args.cell_dimensions = dimensions; + args.n_dimensions = n_dimensions; + + struct cell *cell_data = NULL; + int tmp_ret; + switch (cube->sampling_mode) + { + case SAMPLING_MODE_COMPREHENSIVE: { + cell_data = hash_table_get0_exdata(cube->table, compound_dimension, compound_dimension_len); + if (cell_data == NULL) { + tmp_ret = hash_table_add(cube->table, compound_dimension, compound_dimension_len, (void *)&args); + if (tmp_ret == 1) { + cell_data = hash_table_get0_exdata(cube->table, compound_dimension, compound_dimension_len); + } + } + break;} + case SAMPLING_MODE_TOPK: { + cell_data = heavy_keeper_get0_exdata(cube->heavykeeper, compound_dimension, compound_dimension_len); + if (cell_data == NULL) { + tmp_ret = heavy_keeper_add(cube->heavykeeper, compound_dimension, compound_dimension_len, 0, (void *)&args); + if (tmp_ret == 1) { + cell_data = heavy_keeper_get0_exdata(cube->heavykeeper, compound_dimension, compound_dimension_len); + } + } + break;} + case SAMPLING_MODE_TOP_CARDINALITY: { + cell_data = spread_sketch_get0_exdata(cube->spread_sketch, compound_dimension, compound_dimension_len); + if (cell_data == NULL) { + tmp_ret = spread_sketch_add_hash(cube->spread_sketch, compound_dimension, compound_dimension_len, DUMMY_ITEM_HASH, (void *)&args, DUMMY_TIME_VAL); + if (tmp_ret == 1) { + cell_data = spread_sketch_get0_exdata(cube->spread_sketch, compound_dimension, compound_dimension_len); + } + } + break;} + default: + assert(0); + break; + } + + free(compound_dimension); + return cell_data; +} + +union metric_parameter *construct_parameters(enum metric_type type, ...) +{ + union metric_parameter *paras = (union metric_parameter *)malloc(sizeof(union metric_parameter)); + va_list ap; + va_start(ap, type); + switch (type) { + case METRIC_TYPE_COUNTER: + break; + case METRIC_TYPE_HLL: + paras->hll.precision = (char)va_arg(ap, int); + break; + case METRIC_TYPE_HISTOGRAM: + paras->hdr.lowest_trackable_value = va_arg(ap, long long); + paras->hdr.highest_trackable_value = va_arg(ap, long long); + paras->hdr.significant_figures = va_arg(ap, int); + break; + default: + assert(0); + } + va_end(ap); + return paras; +} + +int cube_register_counter(struct cube *cube, const char *metric_name) { + struct metric_manifest *metric = malloc(sizeof(struct metric_manifest)); + metric->name = strdup(metric_name); + metric->parameters = construct_parameters(METRIC_TYPE_COUNTER); + metric->type = METRIC_TYPE_COUNTER; + + int id = metric_manifest_manager_add(cube->manifest_manager, metric); + if (id < 0) { + free(metric->name); + free(metric->parameters); + free(metric); + return FS_ERR_METRIC_NAME_ALREADY_EXISTS; + } + + metric->id = id; + return id; +} + +int cube_register_hll(struct cube *cube,const char *metric_name, unsigned char precision) { + if (precision < 4 || precision > 18) { + return FS_ERR_INVALID_PARAM; + } + + struct metric_manifest *metric = malloc(sizeof(struct metric_manifest)); + metric->name = strdup(metric_name); + metric->parameters = construct_parameters(METRIC_TYPE_HLL, precision); + metric->type = METRIC_TYPE_HLL; + + int id = metric_manifest_manager_add(cube->manifest_manager, metric); + if (id < 0) { + free(metric->name); + free(metric->parameters); + free(metric); + return FS_ERR_METRIC_NAME_ALREADY_EXISTS; + } + + metric->id = id; + return id; +} + +int cube_register_hist(struct cube *cube,const char *metric_name, long long lowest_trackable_value, long long highest_trackable_value, int significant_figures) { + // refer to hdr_histogram.h for the rules of parameters. Just copy them here + if (lowest_trackable_value < 1) { + return FS_ERR_INVALID_PARAM; + } + if (significant_figures < 1 || significant_figures > 5) { + return FS_ERR_INVALID_PARAM; + } + if (lowest_trackable_value * 2 > highest_trackable_value) + { + return FS_ERR_INVALID_PARAM; + } + + struct metric_manifest *metric = malloc(sizeof(struct metric_manifest)); + metric->name = strdup(metric_name); + metric->parameters = construct_parameters(METRIC_TYPE_HISTOGRAM, lowest_trackable_value, highest_trackable_value, significant_figures); + metric->type = METRIC_TYPE_HISTOGRAM; + + int id = metric_manifest_manager_add(cube->manifest_manager, metric); + if (id < 0) { + free(metric->name); + free(metric->parameters); + free(metric); + return FS_ERR_METRIC_NAME_ALREADY_EXISTS; + } + metric->id = id; + + return id; +} + +int cube_histogram_record(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, long long value) { + if (cube->primary_metric_id == -1) { + return FS_ERR_CUBE_SAMPLING_NOT_INITIALIZED; + } + assert(cube->sampling_mode == SAMPLING_MODE_COMPREHENSIVE || (cube->primary_metric_id != metric_id)); + + const struct metric_manifest *manifest = metric_manifest_manager_get_by_id(cube->manifest_manager, metric_id); + if (manifest == NULL || manifest->type != METRIC_TYPE_HISTOGRAM) { + return FS_ERR_INVALID_METRIC_ID; + } + + struct cell *cell_data = get_cell_in_cube_generic(cube, dimensions, n_dimensions); + if (cell_data == NULL) { + return FS_ERR_TOO_MANY_CELLS; + } + + struct metric *metric = add_or_find_metric_in_cell(manifest, cell_data); + + int ret = metric_histogram_record(metric, value); + if (ret < 0) { + return FS_ERR_INVALID_PARAM; + } + return FS_OK; +} + +int cube_hll_add(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, const char *key, size_t key_len) { + if (cube->primary_metric_id == -1) { + return FS_ERR_CUBE_SAMPLING_NOT_INITIALIZED; + } + + assert(cube->sampling_mode != SAMPLING_MODE_TOPK || cube->primary_metric_id != metric_id); + + const struct metric_manifest *manifest = metric_manifest_manager_get_by_id(cube->manifest_manager, metric_id); + if (manifest == NULL || manifest->type != METRIC_TYPE_HLL) { + return FS_ERR_INVALID_METRIC_ID; + } + + if (cube->sampling_mode == SAMPLING_MODE_TOP_CARDINALITY && cube->primary_metric_id == metric_id) { + char *compound_dimension; + size_t compound_dimension_len; + field_array_to_key(dimensions, n_dimensions, &compound_dimension, &compound_dimension_len); + + struct exdata_new_args args; + args.cell_dimensions = dimensions; + args.n_dimensions = n_dimensions; + + int tmp_ret = spread_sketch_add(cube->spread_sketch, compound_dimension, compound_dimension_len, key, key_len, (void *)&args, DUMMY_TIME_VAL); + free(compound_dimension); + return tmp_ret == 1 ? FS_OK : FS_ERR_TOO_MANY_CELLS; + } + + struct cell *cell_data = get_cell_in_cube_generic(cube, dimensions, n_dimensions); + if (cell_data == NULL) { + return FS_ERR_TOO_MANY_CELLS; + } + + struct metric *metric = add_or_find_metric_in_cell(manifest, cell_data); + metric_hll_add(metric, key, key_len); + return FS_OK; +} + +uint64_t field_array_to_hash(const struct field *field, size_t n_dimensions) { + XXH3_state_t state = {0}; + XXH3_64bits_reset(&state); + + for (int i = 0; i < n_dimensions; i++) { + XXH3_64bits_update(&state, field[i].key, strlen(field[i].key)); + if (field[i].type != FIELD_VALUE_CSTRING) { + XXH3_64bits_update(&state, &field[i].value_longlong, sizeof(long long)); + } else { + XXH3_64bits_update(&state, field[i].value_str, strlen(field[i].value_str)); + } + } + + return XXH3_64bits_digest(&state); +} + +int cube_hll_add_field(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, const struct field *item_fields, size_t n_item) +{ + if (cube->primary_metric_id == -1) { + return FS_ERR_CUBE_SAMPLING_NOT_INITIALIZED; + } + assert(cube->sampling_mode != SAMPLING_MODE_TOPK || (cube->primary_metric_id != metric_id)); + const struct metric_manifest *manifest = metric_manifest_manager_get_by_id(cube->manifest_manager, metric_id); + if (manifest == NULL || manifest->type != METRIC_TYPE_HLL) { + return FS_ERR_INVALID_METRIC_ID; + } + + if (cube->sampling_mode == SAMPLING_MODE_TOP_CARDINALITY && cube->primary_metric_id == metric_id) { + char *compound_dimension; + size_t compound_dimension_len; + field_array_to_key(dimensions, n_dimensions, &compound_dimension, &compound_dimension_len); + + struct exdata_new_args args; + args.cell_dimensions = dimensions; + args.n_dimensions = n_dimensions; + + uint64_t hash = field_array_to_hash(item_fields, n_item); + + int tmp_ret = spread_sketch_add_hash(cube->spread_sketch, compound_dimension, compound_dimension_len, hash, (void *)&args, DUMMY_TIME_VAL); + free(compound_dimension); + return tmp_ret == 1 ? FS_OK : FS_ERR_TOO_MANY_CELLS; + } + + struct cell *cell_data = get_cell_in_cube_generic(cube, dimensions, n_dimensions); + if (cell_data == NULL) { + return FS_ERR_TOO_MANY_CELLS; + } + struct metric *metric = add_or_find_metric_in_cell(manifest, cell_data); + + uint64_t hash = field_array_to_hash(item_fields, n_item); + metric_hll_add_hash(metric, hash); + return FS_OK; +} + +int cube_counter_incrby(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, long long increment) { + if (cube->primary_metric_id == -1) { + return FS_ERR_CUBE_SAMPLING_NOT_INITIALIZED; + } + if (cube->sampling_mode == SAMPLING_MODE_TOPK && cube->primary_metric_id == metric_id && increment < 0) { + return FS_ERR_OPERATION_NOT_SUPPORTED_FOR_PRIMARY_METRIC; + } + assert(cube->sampling_mode == SAMPLING_MODE_COMPREHENSIVE || + (cube->sampling_mode == SAMPLING_MODE_TOPK) || + (cube->sampling_mode == SAMPLING_MODE_TOP_CARDINALITY && cube->primary_metric_id != metric_id) + ); + + const struct metric_manifest *manifest = metric_manifest_manager_get_by_id(cube->manifest_manager, metric_id); + if (manifest == NULL || manifest->type != METRIC_TYPE_COUNTER) { + return FS_ERR_INVALID_METRIC_ID; + } + + if (cube->primary_metric_id == metric_id && cube->sampling_mode == SAMPLING_MODE_TOPK) { + if (increment < 0) { + return FS_ERR_INVALID_PARAM; + } + + char *compound_dimension; + size_t compound_dimension_len; + field_array_to_key(dimensions, n_dimensions, &compound_dimension, &compound_dimension_len); + + struct exdata_new_args args; + args.cell_dimensions = dimensions; + args.n_dimensions = n_dimensions; + + int tmp_ret = heavy_keeper_add(cube->heavykeeper, compound_dimension, compound_dimension_len, increment, (void *)&args); + free(compound_dimension); + return tmp_ret == 1 ? FS_OK : FS_ERR_TOO_MANY_CELLS; + } + + struct cell *cell_data = get_cell_in_cube_generic(cube, dimensions, n_dimensions); + if (cell_data == NULL) { + return FS_ERR_TOO_MANY_CELLS; + } + + struct metric *metric = add_or_find_metric_in_cell(manifest, cell_data); + + metric_counter_incrby(metric, increment); + return FS_OK; +} + +int cube_counter_set(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, long long value) { + if (cube->primary_metric_id == -1) { + return FS_ERR_CUBE_SAMPLING_NOT_INITIALIZED; + } + if (cube->sampling_mode == SAMPLING_MODE_TOPK && cube->primary_metric_id == metric_id) { + return FS_ERR_OPERATION_NOT_SUPPORTED_FOR_PRIMARY_METRIC; + } + + const struct metric_manifest *manifest = metric_manifest_manager_get_by_id(cube->manifest_manager, metric_id); + if (manifest == NULL || manifest->type != METRIC_TYPE_COUNTER) { + return FS_ERR_INVALID_METRIC_ID; + } + + struct cell *cell_data = get_cell_in_cube_generic(cube, dimensions, n_dimensions); + if (cell_data == NULL) { + return FS_ERR_TOO_MANY_CELLS; + } + + struct metric *metric = add_or_find_metric_in_cell(manifest, cell_data); + metric_counter_set(metric, value); + return FS_OK; +} + +struct cube *cube_copy(const struct cube *cube) +{ + struct cube *cube_dup = cube_info_new(cube->cube_dimensions, cube->n_dimensions); + cube_dup->primary_metric_id = cube->primary_metric_id; + cube_dup->sampling_mode = cube->sampling_mode; + cube_dup->max_n_cell = cube->max_n_cell; + + switch (cube->sampling_mode) + { + case SAMPLING_MODE_TOPK: + cube_dup->heavykeeper = heavy_keeper_copy(cube->heavykeeper); + break; + case SAMPLING_MODE_COMPREHENSIVE: + cube_dup->table = hash_table_copy(cube->table); + break; + case SAMPLING_MODE_TOP_CARDINALITY: + cube_dup->spread_sketch = spread_sketch_copy(cube->spread_sketch); + break; + default: + assert(0); + break; + } + + cube_dup->manifest_manager = metric_manifest_manager_copy(cube->manifest_manager); + + return cube_dup; +} + +int cube_merge(struct cube *dest, const struct cube *src) +{ + if (dest->sampling_mode != src->sampling_mode) { + return FS_ERR_DIFFERENT_CONFIGURATION_FOR_SAME_CUBE; + } + if (dest->primary_metric_id == -1 || src->primary_metric_id == -1) { + return FS_ERR_CUBE_SAMPLING_NOT_INITIALIZED; + } + if (dest->primary_metric_id != src->primary_metric_id) { + return FS_ERR_DIFFERENT_CONFIGURATION_FOR_SAME_CUBE; + } + size_t n_metric_src = 0; + const struct metric_manifest **list_src = metric_manifest_manager_list(src->manifest_manager, &n_metric_src); + size_t n_metric_dst = 0; + const struct metric_manifest **list_dst = metric_manifest_manager_list(dest->manifest_manager, &n_metric_dst); + int len_min = n_metric_src < n_metric_dst ? n_metric_src : n_metric_dst; + for (int i = 0; i < len_min; i++) { + if (list_src[i]->type != list_dst[i]->type) { + return FS_ERR_DIFFERENT_CONFIGURATION_FOR_SAME_CUBE; + } + if (strcmp(list_src[i]->name, list_dst[i]->name) != 0) { + return FS_ERR_DIFFERENT_CONFIGURATION_FOR_SAME_CUBE; + } + } + + for (int i = n_metric_dst; i < n_metric_src; i++) { + metric_manifest_manager_add(dest->manifest_manager, metric_manifest_copy(list_src[i])); + } + + switch (dest->sampling_mode) + { + case SAMPLING_MODE_TOPK: + heavy_keeper_merge(dest->heavykeeper, src->heavykeeper); + break; + case SAMPLING_MODE_COMPREHENSIVE: + hash_table_merge(dest->table, src->table); + break; + case SAMPLING_MODE_TOP_CARDINALITY: + spread_sketch_merge(dest->spread_sketch, src->spread_sketch); + break; + default: + assert(0); + break; + } + + return FS_OK; +} + +struct cube *cube_fork(const struct cube *cube) { + struct cube *ret = cube_info_new(cube->cube_dimensions, cube->n_dimensions); + ret->primary_metric_id = cube->primary_metric_id; + ret->sampling_mode = cube->sampling_mode; + ret->max_n_cell = cube->max_n_cell; + ret->manifest_manager = metric_manifest_manager_copy(cube->manifest_manager); + + switch (cube->sampling_mode) { + case SAMPLING_MODE_TOPK: + ret->heavykeeper = heavy_keeper_new(cube->max_n_cell); + heavy_keeper_set_exdata_schema(ret->heavykeeper, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i); + break; + case SAMPLING_MODE_COMPREHENSIVE: + ret->table = hash_table_new(cube->max_n_cell); + hash_table_set_exdata_schema(ret->table, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i); + break; + case SAMPLING_MODE_TOP_CARDINALITY: { + int width, depth, dummy_time; + unsigned char precision; + spread_sketch_get_parameter(cube->spread_sketch, &depth, &width, &precision, &dummy_time); + ret->spread_sketch = spread_sketch_new(depth, width, precision, 0, DUMMY_TIME_VAL); + spread_sketch_set_exdata_schema(ret->spread_sketch, exdata_new_i, exdata_free_i, exdata_merge_i, exdata_reset_i, exdata_copy_i); + break;} + default: + assert(0); + break; + } + return ret; +} + +/* -------------------------------------------------------------------------- */ +/* query */ +/* -------------------------------------------------------------------------- */ + +struct tmp_sorted_data_spread_sketch_cell { + double hll_value; + struct cell *data; +}; +static int compare_tmp_sorted_data_spread_sketch_cell(const void *a, const void *b) { // sort in descending order + const struct tmp_sorted_data_spread_sketch_cell *aa = (const struct tmp_sorted_data_spread_sketch_cell *)a; + const struct tmp_sorted_data_spread_sketch_cell *bb = (const struct tmp_sorted_data_spread_sketch_cell *)b; + if (aa->hll_value < bb->hll_value) { + return 1; + } else if (aa->hll_value > bb->hll_value) { + return -1; + } else { + return 0; + } +} +void cube_get_cells(const struct cube *cube, struct field_list **cell_dimensions, size_t *n_cell) +{ + size_t n_cell_tmp = cube_get_cell_count(cube); + if (n_cell_tmp == 0) { + *cell_dimensions = NULL; + *n_cell = 0; + return; + } + + char **spread_sketch_keys = NULL; + size_t *spread_sketch_keys_lens = NULL; + struct cell **cell_datas = (struct cell **)malloc(sizeof(struct cell *) * n_cell_tmp); + switch (cube->sampling_mode) { + case SAMPLING_MODE_COMPREHENSIVE: + hash_table_list(cube->table, (void **)cell_datas, n_cell_tmp); + break; + case SAMPLING_MODE_TOPK: + heavy_keeper_list(cube->heavykeeper, (void **)cell_datas, n_cell_tmp); + break; + case SAMPLING_MODE_TOP_CARDINALITY: { + spread_sketch_list_keys(cube->spread_sketch, &spread_sketch_keys, &spread_sketch_keys_lens, &n_cell_tmp); + for (int i = 0; i < n_cell_tmp; i++) { + cell_datas[i] = spread_sketch_get0_exdata(cube->spread_sketch, spread_sketch_keys[i], spread_sketch_keys_lens[i]); + } + } + break; + default: + assert(0); + } + + // spread sketch often stores more than max_n_cell. So sort out the top max_n_cell cells. + if (cube->sampling_mode == SAMPLING_MODE_TOP_CARDINALITY && n_cell_tmp > cube->max_n_cell) { + struct tmp_sorted_data_spread_sketch_cell *tmp_sorted_data = (struct tmp_sorted_data_spread_sketch_cell *)malloc(sizeof(struct tmp_sorted_data_spread_sketch_cell) * n_cell_tmp); + for (int i = 0; i < n_cell_tmp; i++) { + tmp_sorted_data[i].data = cell_datas[i]; + tmp_sorted_data[i].hll_value = spread_sketch_get_cardinality(cube->spread_sketch, spread_sketch_keys[i], spread_sketch_keys_lens[i]); + } + qsort(tmp_sorted_data, n_cell_tmp, sizeof(struct tmp_sorted_data_spread_sketch_cell), compare_tmp_sorted_data_spread_sketch_cell); + + free(cell_datas); + cell_datas = (struct cell **)malloc(sizeof(struct cell *) * cube->max_n_cell); + for (int i = 0; i < cube->max_n_cell; i++) { + cell_datas[i] = tmp_sorted_data[i].data; + } + n_cell_tmp = cube->max_n_cell; + free(tmp_sorted_data); + } + + struct field_list *ret = (struct field_list *)malloc(sizeof(struct field_list) * n_cell_tmp); + *cell_dimensions = ret; + *n_cell = n_cell_tmp; + for (int i = 0; i < n_cell_tmp; i++) { + struct cell *cell_data = cell_datas[i]; + struct field_list *field_list_tmp = &ret[i]; + field_list_tmp->n_field = cell_data->cell_dimensions.n_field; + if (field_list_tmp->n_field == 0) { + field_list_tmp->field = NULL; + continue; + } + field_list_tmp->field = field_array_duplicate(cell_data->cell_dimensions.field, field_list_tmp->n_field); + } + + free(cell_datas); + free(spread_sketch_keys); + free(spread_sketch_keys_lens); +} + +const struct cell *cube_find_cell_by_dimension(const struct cube *cube, const struct field_list *fields) +{ + const struct cell *ret = NULL; + char *compound_dimension; + size_t dimension_len; + field_array_to_key(fields->field, fields->n_field, &compound_dimension, &dimension_len); + + switch (cube->sampling_mode) + { + case SAMPLING_MODE_TOPK: + ret = heavy_keeper_get0_exdata(cube->heavykeeper, compound_dimension, dimension_len); + break; + case SAMPLING_MODE_COMPREHENSIVE: + ret = hash_table_get0_exdata(cube->table, compound_dimension, dimension_len); + break; + case SAMPLING_MODE_TOP_CARDINALITY: + ret = spread_sketch_get0_exdata(cube->spread_sketch, compound_dimension, dimension_len); + break; + default: + assert(0); + return NULL; + } + free(compound_dimension); + + return ret; +} + +const struct metric *cube_find_uncleared_metric_in_cell(const struct cube *cube, const struct field_list *fields, int metric_id,int *ret_code) +{ + const struct cell *data = cube_find_cell_by_dimension(cube, fields); + + if (data == NULL) { + *ret_code = FS_ERR_INVALID_DIMENSION; + return NULL; + } + + if (metric_id < 0 || metric_id >= data->next_index) { + *ret_code = FS_ERR_INVALID_METRIC_ID; + return NULL; + } + *ret_code = FS_OK; + + const struct metric *ret_metric = data->slots[metric_id]; + if (ret_metric == NULL || metric_check_if_cleared(ret_metric)) { + *ret_code = FS_ERR_INVALID_METRIC_ID; + return NULL; + } + + return ret_metric; +} + +int cube_counter_get(const struct cube *cube, int metric_id, const struct field_list *fields, long long *value) +{ + if (cube->sampling_mode == SAMPLING_MODE_TOPK && cube->primary_metric_id == metric_id) { + char *dimension_in_string; + size_t dimension_string_len; + field_array_to_key(fields->field, fields->n_field, &dimension_in_string, &dimension_string_len); + + long long count = 0; + void *exdata_dummy = NULL; + heavy_keeper_one_point_query(cube->heavykeeper, dimension_in_string, dimension_string_len, &count, &exdata_dummy); + *value = count; + + free(dimension_in_string); + return count == 0 ? FS_ERR_INVALID_DIMENSION : FS_OK; + } + + int ret; + const struct metric *metric = cube_find_uncleared_metric_in_cell(cube, fields, metric_id, &ret); + if (ret != FS_OK) { + return ret; + } + if (metric == NULL) { + return FS_ERR_INVALID_METRIC_ID; + } + + *value = metric_counter_get(metric); + return FS_OK; +} + +int cube_hll_get(const struct cube *cube, int metric_id, const struct field_list *fields, double *value) +{ + if (cube->sampling_mode == SAMPLING_MODE_TOP_CARDINALITY && cube->primary_metric_id == metric_id) { + char *dimension_in_string; + size_t dimension_string_len; + field_array_to_key(fields->field, fields->n_field, &dimension_in_string, &dimension_string_len); + + double hll_value = spread_sketch_get_cardinality(cube->spread_sketch, dimension_in_string, dimension_string_len); + free(dimension_in_string); + if (hll_value < 0) { + return FS_ERR_INVALID_DIMENSION; + } + + *value = hll_value; + return FS_OK; + } + + int ret; + const struct metric *metric = cube_find_uncleared_metric_in_cell(cube, fields, metric_id, &ret); + if (ret != FS_OK) { + return ret; + } + if (metric == NULL) { + return FS_ERR_INVALID_METRIC_ID; + } + + *value = metric_hll_get(metric); + return FS_OK; +} + +int cube_histogram_value_at_percentile(const struct cube *cube, int metric_id, const struct field_list *fields, double percentile, long long *value) +{ + int ret; + const struct metric *metric = cube_find_uncleared_metric_in_cell(cube, fields, metric_id, &ret); + if (ret != FS_OK) { + return ret; + } + if (metric == NULL) { + return FS_ERR_INVALID_METRIC_ID; + } + + *value = metric_histogram_value_at_percentile(metric, percentile); + return FS_OK; +} + +int cube_histogram_count_le_value(const struct cube *cube, int metric_id, const struct field_list *fields, long long value, long long *count) { + int ret; + const struct metric *metric = cube_find_uncleared_metric_in_cell(cube, fields, metric_id, &ret); + if (ret != FS_OK) { + return ret; + } + if (metric == NULL) { + return FS_ERR_INVALID_METRIC_ID; + } + + *count = metric_histogram_count_le_value(metric, value); + return FS_OK; +} + +int cube_get_serialization_as_base64(const struct cube *cube, int metric_id, const struct field_list *fields, char **blob, size_t *blob_size) { + if (cube->sampling_mode == SAMPLING_MODE_TOP_CARDINALITY && cube->primary_metric_id == metric_id) { + char *dimension_in_string; + size_t dimension_string_len; + field_array_to_key(fields->field, fields->n_field, &dimension_in_string, &dimension_string_len); + + *blob = spread_sketch_get_hll_base64_serialization(cube->spread_sketch, dimension_in_string, dimension_string_len); + *blob_size = strlen(*blob); + free(dimension_in_string); + return FS_OK; + } + + int ret; + const struct metric *metric = cube_find_uncleared_metric_in_cell(cube, fields, metric_id, &ret); + if (ret != FS_OK) { + return ret; + } + if (metric == NULL) { + return FS_ERR_INVALID_METRIC_ID; + } + + metric_serialize(metric, blob, blob_size); + return FS_OK; +} + +int cube_get_cell_count(const struct cube *cube) { + switch (cube->sampling_mode) { + case SAMPLING_MODE_COMPREHENSIVE: + return hash_table_get_count(cube->table); + case SAMPLING_MODE_TOPK: + return heavy_keeper_get_count(cube->heavykeeper); + case SAMPLING_MODE_TOP_CARDINALITY: + return spread_sketch_get_count(cube->spread_sketch); + default: + assert(0); + return -1; // to mute cppcheck + } +} + +void cube_get_metrics_in_cell(const struct cube *cube, const struct field_list *fields, int **metric_id_out, size_t *n_metric_out) { + const struct cell *cell_data = cube_find_cell_by_dimension(cube, fields); + if (cell_data == NULL) { + *metric_id_out = NULL; + *n_metric_out = 0; + return; + } + + *metric_id_out = (int *)malloc(sizeof(int) * cell_data->next_index + 1); // +1: for primary metric + int n_metric = 0; + if (cube->sampling_mode == SAMPLING_MODE_TOP_CARDINALITY || cube->sampling_mode == SAMPLING_MODE_TOPK) { // primary metric is not stored in cell_data + (*metric_id_out)[n_metric] = cube->primary_metric_id; + n_metric++; + } + + for (int i = 0; i < cell_data->next_index; i++) { + if (cell_data->slots[i] != NULL && !metric_check_if_cleared(cell_data->slots[i])) { + (*metric_id_out)[n_metric] = i; + n_metric++; + } + } + *n_metric_out = n_metric; +} + +struct field_list *cube_get_identifier(const struct cube *cube) { + struct field_list *ret = (struct field_list *)malloc(sizeof(struct field_list)); + + if (cube->n_dimensions == 0) { + ret->field = NULL; + ret->n_field = 0; + return ret; + } + + ret->field = field_array_duplicate(cube->cube_dimensions, cube->n_dimensions); + ret->n_field = cube->n_dimensions; + + + return ret; +} + +const char *cube_get_metric_name(const struct cube *cube, int metric_id) { + const struct metric_manifest *metric = metric_manifest_manager_get_by_id(cube->manifest_manager, metric_id); + if (metric == NULL) { + return NULL; + } + + return metric->name; +} + +enum metric_type cube_get_metric_type(const struct cube *cube, int metric_id) { + const struct metric_manifest *metric = metric_manifest_manager_get_by_id(cube->manifest_manager, metric_id); + if (metric == NULL) { + return (enum metric_type)(-1); + } + + return metric->type; +} + +void cube_get_metrics(const struct cube *cube, int **metric_id_out, size_t *n_metric) +{ + const struct metric_manifest **list = metric_manifest_manager_list(cube->manifest_manager, n_metric); + if (*n_metric == 0) { + *metric_id_out = NULL; + return; + } + + int *tmp_ids = (int *)malloc(sizeof(int) * (*n_metric)); + *metric_id_out = tmp_ids; + + for (int i = 0; i < *n_metric; i++) { + tmp_ids[i] = list[i]->id; + } +} diff --git a/src/cube.h b/src/cube.h new file mode 100644 index 0000000..1134aa6 --- /dev/null +++ b/src/cube.h @@ -0,0 +1,71 @@ +#pragma once +#include <stdio.h> +#ifdef __cplusplus +extern "C" +{ +#endif + +#include <stddef.h> +#include <stdbool.h> +#include "fieldstat.h" // for dimensions +#include "metric_manifest.h" + +struct cube; +struct cube_manager; + +struct cube *cube_new(const struct field *dimensions, size_t n_dimensions); +int cube_set_sampling(struct cube *cube, enum sampling_mode mode, int max_n_cell, int primary_metric_id); +void cube_free(struct cube *cube); +void cube_reset(struct cube *cube); +struct cube *cube_copy(const struct cube *cube); +int cube_merge(struct cube *dest, const struct cube *src); +struct cube *cube_fork(const struct cube *cube); // only copy the cube configurations, leave the cells empty + +int cube_register_counter(struct cube *cube, const char *metric_name); +int cube_register_hll(struct cube *cube,const char *metric_name, unsigned char precision); +int cube_register_hist(struct cube *cube,const char *metric_name, long long lowest_trackable_value, long long highest_trackable_value, int significant_figures); + +int cube_histogram_record(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, long long value); +int cube_hll_add(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, const char *key, size_t key_len); +int cube_hll_add_field(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, const struct field *item_fields, size_t n_item); +int cube_counter_incrby(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, long long increment); +int cube_counter_set(struct cube *cube, int metric_id, const struct field *dimensions, size_t n_dimensions, long long value); + +/* ---------------------------------- query --------------------------------- */ +int cube_counter_get(const struct cube *cube, int metric_id, const struct field_list *dimensions, long long *value); +int cube_hll_get(const struct cube *cube, int metric_id, const struct field_list *dimensions, double *value); +int cube_histogram_value_at_percentile(const struct cube *cube, int metric_id, const struct field_list *dimensions, double percentile, long long *value); +int cube_histogram_count_le_value(const struct cube *cube, int metric_id, const struct field_list *dimensions, long long value, long long *count); +int cube_get_serialization_as_base64(const struct cube *cube, int metric_id, const struct field_list *dimensions, char **blob, size_t *blob_size); + +int cube_get_cell_count(const struct cube *cube); +enum sampling_mode cube_get_sampling_mode(const struct cube *cube); +void cube_get_cells(const struct cube *cube, struct field_list **cell_dimensions, size_t *n_cell); +void cube_get_metrics(const struct cube *cube, int **metric_id_out, size_t *n_metric); +void cube_get_metrics_in_cell(const struct cube *cube, const struct field_list *dimensions, int **metric_id_out, size_t *n_metric_out); +struct field_list *cube_get_identifier(const struct cube *cube); +const char *cube_get_metric_name(const struct cube *cube, int metric_id); +enum metric_type cube_get_metric_type(const struct cube *cube, int metric_id); + +/* -------------------------------------------------------------------------- */ +/* cube manager */ +/* -------------------------------------------------------------------------- */ +struct cube *cube_manager_get_cube_by_id(const struct cube_manager *manager, int cube_id); + +// the cube will be taken over by the manager, user do not free it. +int cube_manager_add(struct cube_manager *pthis, struct cube *cube); +void cube_manager_delete(struct cube_manager *pthis, struct cube *cube); // the cube will be freed by the manager +int cube_manager_find(const struct cube_manager *pthis, const struct field *cube_dimensions, size_t n_dimensions); +struct cube_manager *cube_manager_new(); +void cube_manager_free(struct cube_manager *pthis); +int cube_manager_merge(struct cube_manager *dest, const struct cube_manager *src); + +void cube_manager_reset(struct cube_manager *pthis); +void cube_manager_calibrate(struct cube_manager *pthis, const struct cube_manager *master); +struct cube_manager *cube_manager_fork(const struct cube_manager *src); + +void cube_manager_list(const struct cube_manager *pthis, int **cube_ids, int *n_cube); + +#ifdef __cplusplus +} +#endif
\ No newline at end of file diff --git a/src/exdata.h b/src/exdata.h new file mode 100644 index 0000000..a9c28de --- /dev/null +++ b/src/exdata.h @@ -0,0 +1,7 @@ +#pragma once + +typedef void * (*exdata_new_cb)(void *arg); +typedef void (*exdata_free_cb)(void *exdata); +typedef void (*exdata_merge_cb)(void *dest, void *src); +typedef void (*exdata_reset_cb)(void *exdata); +typedef void * (*exdata_copy_cb)(void *exdata); diff --git a/src/exporter/cjson_exporter.c b/src/exporter/cjson_exporter.c index 206f7b1..5954a1e 100644 --- a/src/exporter/cjson_exporter.c +++ b/src/exporter/cjson_exporter.c @@ -46,11 +46,12 @@ ] */ #define DEFAULT_EXPORTER_NAME "-" +#define NIL_TAG_JSON "\anil" struct counter_history; struct export_kv_pair { char *key; - enum fs_tag_type type; + enum field_type type; union{ long long value_longlong; double value_double; @@ -60,7 +61,7 @@ struct export_kv_pair { struct fieldstat_json_exporter { char *name; - struct fieldstat_tag_list *global_tag_list; + struct field_list *global_tag_list; struct counter_history *history; long long last_ts; }; @@ -69,20 +70,17 @@ struct cell_iter { int *cube_ids; int n_cube; int curr_cube_idx; - struct fieldstat_tag_list *shared_tag; + struct field_list *cube_dimension; - struct fieldstat_tag_list *tag_list; + struct field_list *cell_dimension; size_t n_cell; size_t curr_cell_idx; - int *metric_ids; - size_t n_metric; - const struct fieldstat *instance; }; struct cellwise_rec_for_export { - struct json_writer *cjson_tags; + struct json_writer *tag_json_obj; struct export_kv_pair **metric_pairs; size_t n_metric; }; @@ -103,12 +101,8 @@ struct counter_history { struct tag_metric_map *rec; long long ts; - unsigned long cell_version; - unsigned long *cube_version; - size_t n_cube; - char *exporter_name; - struct fieldstat_tag_list *global_tag_list; + struct field_list *global_tag_list; }; struct couple_export_table_item { @@ -123,25 +117,25 @@ struct couple_export_table { void kv_pair_free(struct export_kv_pair *pair) { - if (pair->type == TAG_CSTRING) { + if (pair->type == FIELD_VALUE_CSTRING) { free((char *)pair->value_str); } free((char *)pair->key); free(pair); } -void kv_pair_fill_with_tags(struct export_kv_pair *dest, const struct fieldstat_tag *src) +void kv_pair_fill_with_fields(struct export_kv_pair *dest, const struct field *src) { dest->key = strdup(src->key); dest->type = src->type; switch (src->type) { - case TAG_INTEGER: + case FIELD_VALUE_INTEGER: dest->value_longlong = src->value_longlong; break; - case TAG_DOUBLE: + case FIELD_VALUE_DOUBLE: dest->value_double = src->value_double; break; - case TAG_CSTRING: + case FIELD_VALUE_CSTRING: dest->value_str = strdup(src->value_str); break; default: @@ -212,44 +206,42 @@ void counter_history_free(struct counter_history *history) free(tag_node); } - free(history->cube_version); - free(history->exporter_name); if (history->global_tag_list != NULL) { - fieldstat_tag_list_arr_free(history->global_tag_list, 1); + fieldstat_field_list_arr_free(history->global_tag_list, 1); } free(history); } -bool fieldstat_tag_list_cmp(const struct fieldstat_tag_list *a, const struct fieldstat_tag_list *b) +bool field_list_cmp(const struct field_list *a, const struct field_list *b) { - if (a->n_tag != b->n_tag) { + if (a->n_field != b->n_field) { return false; } - for (int i = 0; i < a->n_tag; i++) { - if (strcmp(a->tag[i].key, b->tag[i].key) != 0) { + for (int i = 0; i < a->n_field; i++) { + if (strcmp(a->field[i].key, b->field[i].key) != 0) { return false; } - if (a->tag[i].type != b->tag[i].type) { + if (a->field[i].type != b->field[i].type) { return false; } - switch (a->tag[i].type) + switch (a->field[i].type) { - case TAG_INTEGER: - if (a->tag[i].value_longlong != b->tag[i].value_longlong) { + case FIELD_VALUE_INTEGER: + if (a->field[i].value_longlong != b->field[i].value_longlong) { return false; } break; - case TAG_DOUBLE: - if (a->tag[i].value_double != b->tag[i].value_double) { + case FIELD_VALUE_DOUBLE: + if (a->field[i].value_double != b->field[i].value_double) { return false; } break; - case TAG_CSTRING: - if (strcmp(a->tag[i].value_str, b->tag[i].value_str) != 0) { + case FIELD_VALUE_CSTRING: + if (strcmp(a->field[i].value_str, b->field[i].value_str) != 0) { return false; } break; @@ -261,24 +253,24 @@ bool fieldstat_tag_list_cmp(const struct fieldstat_tag_list *a, const struct fie return true; } -struct fieldstat_tag_list *my_copy_fs_tag_list(const struct fieldstat_tag_list *src) +struct field_list *field_list_dup(const struct field_list *src) { - struct fieldstat_tag_list *dest = malloc(sizeof(struct fieldstat_tag_list)); - dest->n_tag = src->n_tag; - dest->tag = malloc(sizeof(struct fieldstat_tag) * dest->n_tag); - for (int i = 0; i < dest->n_tag; i++) { - dest->tag[i].key = strdup(src->tag[i].key); - dest->tag[i].type = src->tag[i].type; - switch (src->tag[i].type) + struct field_list *dest = malloc(sizeof(struct field_list)); + dest->n_field = src->n_field; + dest->field = malloc(sizeof(struct field) * dest->n_field); + for (int i = 0; i < dest->n_field; i++) { + dest->field[i].key = strdup(src->field[i].key); + dest->field[i].type = src->field[i].type; + switch (src->field[i].type) { - case TAG_INTEGER: - dest->tag[i].value_longlong = src->tag[i].value_longlong; + case FIELD_VALUE_INTEGER: + dest->field[i].value_longlong = src->field[i].value_longlong; break; - case TAG_DOUBLE: - dest->tag[i].value_double = src->tag[i].value_double; + case FIELD_VALUE_DOUBLE: + dest->field[i].value_double = src->field[i].value_double; break; - case TAG_CSTRING: - dest->tag[i].value_str = strdup(src->tag[i].value_str); + case FIELD_VALUE_CSTRING: + dest->field[i].value_str = strdup(src->field[i].value_str); break; default: assert(0); @@ -290,21 +282,16 @@ struct fieldstat_tag_list *my_copy_fs_tag_list(const struct fieldstat_tag_list * bool counter_history_check_if_need_to_update(const struct fieldstat_json_exporter *exporter, const struct fieldstat *instance) { - struct counter_history *history = exporter->history; + const struct counter_history *history = exporter->history; if (history == NULL) { return false; // delta export disabled } - if (history->cube_version == NULL) { // first time + if (history->exporter_name == NULL) { // first time return true; } - unsigned long cur_cell_version = fieldstat_get_version(instance); const char *cur_exporter_name = exporter->name ? exporter->name : DEFAULT_EXPORTER_NAME; - const struct fieldstat_tag_list *cur_global_tag_list = exporter->global_tag_list; - - if (history->cell_version != cur_cell_version) { - return true; - } + const struct field_list *cur_global_tag_list = exporter->global_tag_list; if (strcmp(history->exporter_name, cur_exporter_name) != 0) { return true; @@ -316,51 +303,24 @@ bool counter_history_check_if_need_to_update(const struct fieldstat_json_exporte if (history->global_tag_list == NULL && cur_global_tag_list == NULL) { return false; } - // global tag cant be deleted, so no need to check if cur_global_tag_list == NULL && history->global_tag_list != NULL - if (fieldstat_tag_list_cmp(cur_global_tag_list, history->global_tag_list) == false) { - return true; - } - - int *cube_ids = NULL; - int n_cube = 0; - fieldstat_get_cubes(instance, &cube_ids, &n_cube); - if (n_cube != history->n_cube) { + // global field cant be deleted, so no need to check if cur_global_tag_list == NULL && history->global_tag_list != NULL + if (field_list_cmp(cur_global_tag_list, history->global_tag_list) == false) { return true; } - for (int i = 0; i < n_cube; i++) { - unsigned long cube_version = fieldstat_get_cube_version(instance, cube_ids[i]); - if (cube_version != history->cube_version[i]) { - return true; - } - } - free(cube_ids); return false; } void counter_history_fill_version_info(struct counter_history *history, struct fieldstat_json_exporter *exporter, const struct fieldstat *instance) { - unsigned long cur_cell_version = fieldstat_get_version(instance); const char *cur_exporter_name = exporter->name ? exporter->name : DEFAULT_EXPORTER_NAME; - const struct fieldstat_tag_list *cur_global_tag_list = exporter->global_tag_list; + const struct field_list *cur_global_tag_list = exporter->global_tag_list; free(history->exporter_name); history->exporter_name = strdup(cur_exporter_name); if (cur_global_tag_list != NULL) { - history->global_tag_list = my_copy_fs_tag_list(cur_global_tag_list); - } - - history->cell_version = cur_cell_version; - - int *cube_ids = NULL; - int n_cube = 0; - fieldstat_get_cubes(instance, &cube_ids, &n_cube); - history->n_cube = n_cube; - history->cube_version = malloc(sizeof(unsigned long) * n_cube); - for (int i = 0; i < n_cube; i++) { - history->cube_version[i] = fieldstat_get_cube_version(instance, cube_ids[i]); + history->global_tag_list = field_list_dup(cur_global_tag_list); } - free(cube_ids); } void fieldstat_json_exporter_update_history(struct fieldstat_json_exporter *exporter, const struct fieldstat *instance) @@ -377,13 +337,13 @@ void fieldstat_json_exporter_update_history(struct fieldstat_json_exporter *expo void write_delta_to_json(struct fieldstat_json_exporter *exporter, struct cellwise_rec_for_export *tag_field_pair, struct json_writer *field_json) { - // for every tag_field_pair, get the tag json string - const char *tag_json = json_writer_unwrap(tag_field_pair->cjson_tags); + // for every tag_field_pair, get the field json string + const char *tag_json = json_writer_unwrap(tag_field_pair->tag_json_obj); if (tag_json == NULL) { - tag_json = "\a\t\a"; // just a dummy string + tag_json = NIL_TAG_JSON; } for (int j = 0; j < tag_field_pair->n_metric; j++) { - if (tag_field_pair->metric_pairs[j]->type != TAG_INTEGER) { // only counter type need to write delta + if (tag_field_pair->metric_pairs[j]->type != FIELD_VALUE_INTEGER) { // only counter type need to write delta continue; } const char *metric_name = tag_field_pair->metric_pairs[j]->key; @@ -448,15 +408,13 @@ void couple_export_table_free(struct couple_export_table *tbl) free(tbl); } -#define NIL_TAG_JSON "\anil" - -// align delta fields to acc fields by matching tag json string, return an array with length n_acc, each of whose element match the corresponding acc element by both tag and index +// align delta fields to acc fields by matching field json string, return an array with length n_acc, each of whose element match the corresponding acc element by both field and index const struct cellwise_rec_for_export **rearrange_metric_delta(const struct cellwise_rec_for_export *acc, size_t n_acc, const struct cellwise_rec_for_export *delta, size_t n_delta) { struct couple_export_table *map = couple_export_table_new(); for (int i = 0; i < n_delta; i++) { - const char *tag_json = json_writer_unwrap(delta[i].cjson_tags); - if (tag_json == NULL) { // tag might be empty, give it a dummy name + const char *tag_json = json_writer_unwrap(delta[i].tag_json_obj); + if (tag_json == NULL) { // field might be empty, give it a dummy name tag_json = NIL_TAG_JSON; } couple_export_table_add(map, tag_json, i); @@ -465,8 +423,8 @@ const struct cellwise_rec_for_export **rearrange_metric_delta(const struct cellw const struct cellwise_rec_for_export **ret = calloc(n_acc, sizeof(struct cellwise_rec_for_export *)); for (int id_acc = 0; id_acc < n_acc; id_acc++) { - const char *tag_str_acc = json_writer_unwrap(acc[id_acc].cjson_tags); - if (tag_str_acc == NULL) { // tag might be empty, give it a dummy name + const char *tag_str_acc = json_writer_unwrap(acc[id_acc].tag_json_obj); + if (tag_str_acc == NULL) { // field might be empty, give it a dummy name tag_str_acc = NIL_TAG_JSON; } @@ -495,8 +453,8 @@ struct cell_iter *cell_iter_new(const struct fieldstat *instance) { } void cell_iter_free_cell_records(struct cell_iter *iter) { - fieldstat_tag_list_arr_free(iter->tag_list, iter->n_cell); - iter->tag_list = NULL; + fieldstat_field_list_arr_free(iter->cell_dimension, iter->n_cell); + iter->cell_dimension = NULL; iter->n_cell = 0; iter->curr_cell_idx = 0; } @@ -504,11 +462,10 @@ void cell_iter_free_cell_records(struct cell_iter *iter) { void cell_iter_free(struct cell_iter *iter) { cell_iter_free_cell_records(iter); - if (iter->shared_tag != NULL) { - fieldstat_tag_list_arr_free(iter->shared_tag, 1); + if (iter->cube_dimension != NULL) { + fieldstat_field_list_arr_free(iter->cube_dimension, 1); } free(iter->cube_ids); - free(iter->metric_ids); free(iter); } @@ -518,20 +475,12 @@ int cell_iter_next_cube(struct cell_iter *iter) { while (iter->curr_cube_idx < iter->n_cube - 1) { int cube_id_next = iter->cube_ids[++iter->curr_cube_idx]; - fieldstat_get_metrics_used_by_cube(instance, cube_id_next, &iter->metric_ids, &iter->n_metric); - if (iter->n_metric == 0) { - continue; - } - - // get cell info - fieldstat_get_cells_used_by_cube(instance, cube_id_next, &iter->tag_list, &iter->n_cell); + fieldstat_cube_get_cells(instance, cube_id_next, &iter->cell_dimension, &iter->n_cell); if (iter->n_cell == 0) { - free(iter->metric_ids); - iter->metric_ids = NULL; continue; } - iter->shared_tag = fieldstat_get_shared_tags(iter->instance, iter->cube_ids[iter->curr_cube_idx]); + iter->cube_dimension = fieldstat_cube_get_dimensions(iter->instance, iter->cube_ids[iter->curr_cube_idx]); iter->curr_cell_idx = 0; return 1; } @@ -560,10 +509,9 @@ int cell_iter_next(struct cell_iter *iter) { } cell_iter_free_cell_records(iter); - fieldstat_tag_list_arr_free(iter->shared_tag, 1); - iter->shared_tag = NULL; - free(iter->metric_ids); - iter->metric_ids = NULL; + fieldstat_field_list_arr_free(iter->cube_dimension, 1); + iter->cube_dimension = NULL; + if (cell_iter_next_cube(iter)) { return 1; } @@ -575,30 +523,31 @@ int cell_iter_next(struct cell_iter *iter) { /* -------------------------------------------------------------------------- */ struct export_kv_pair *cell_query_with_iter(const struct cell_iter *iter, int metric_id) { - enum metric_type type = fieldstat_get_metric_type(iter->instance, metric_id); + int cube_id = iter->cube_ids[iter->curr_cube_idx]; + enum metric_type type = fieldstat_get_metric_type(iter->instance, cube_id, metric_id); struct export_kv_pair *ret = NULL; if (type == METRIC_TYPE_COUNTER) { long long value; - int tmp_ret = fieldstat_counter_get(iter->instance, iter->cube_ids[iter->curr_cube_idx], metric_id, &iter->tag_list[iter->curr_cell_idx], &value); + int tmp_ret = fieldstat_counter_get(iter->instance, cube_id, &iter->cell_dimension[iter->curr_cell_idx], metric_id, &value); if (tmp_ret < 0) { return NULL; } ret = malloc(sizeof(struct export_kv_pair)); - ret->key = strdup(fieldstat_get_metric_name(iter->instance, metric_id)); - ret->type = TAG_INTEGER; + ret->key = strdup(fieldstat_get_metric_name(iter->instance, cube_id, metric_id)); + ret->type = FIELD_VALUE_INTEGER; ret->value_longlong = value; return ret; } if (type == METRIC_TYPE_HLL || type == METRIC_TYPE_HISTOGRAM) { char *value; size_t len; - fieldstat_get_serialized_blob(iter->instance, iter->cube_ids[iter->curr_cube_idx], metric_id, &iter->tag_list[iter->curr_cell_idx], &value, &len); + fieldstat_metric_get_serialization_as_base64(iter->instance, cube_id, metric_id, &iter->cell_dimension[iter->curr_cell_idx], &value, &len); if (value == NULL) { return NULL; } ret = malloc(sizeof(struct export_kv_pair)); - ret->key = strdup(fieldstat_get_metric_name(iter->instance, metric_id)); - ret->type = TAG_CSTRING; + ret->key = strdup(fieldstat_get_metric_name(iter->instance, cube_id, metric_id)); + ret->type = FIELD_VALUE_CSTRING; ret->value_str = value; return ret; } @@ -610,13 +559,13 @@ void kv_pair_write_to_json(const struct export_kv_pair *pairs, struct json_write { switch (pairs->type) { - case TAG_INTEGER: + case FIELD_VALUE_INTEGER: json_writer_longlong_field(writer, pairs->key, pairs->value_longlong); break; - case TAG_DOUBLE: + case FIELD_VALUE_DOUBLE: json_writer_double_field(writer, pairs->key, pairs->value_double); break; - case TAG_CSTRING: + case FIELD_VALUE_CSTRING: json_writer_str_field(writer, pairs->key, pairs->value_str, strlen(pairs->value_str)); break; default: @@ -624,52 +573,51 @@ void kv_pair_write_to_json(const struct export_kv_pair *pairs, struct json_write } } -void tag_list_append_to_tag_object(const struct fieldstat_tag_list *tag_list, struct json_writer *cjson_tags) +void field_list_write_to_json(const struct field_list *field_list, struct json_writer *json_obj) { - size_t tag_list_len = tag_list->n_tag; struct export_kv_pair pairs = {0}; - for (int i = 0; i < tag_list_len; i++) { - if (tag_list->n_tag == 0) { + for (int i = 0; i < field_list->n_field; i++) { + if (field_list->n_field == 0) { continue; } memset(&pairs, 0, sizeof(struct export_kv_pair)); - pairs.key = (char *)tag_list->tag[i].key; - pairs.type = tag_list->tag[i].type; + pairs.key = (char *)field_list->field[i].key; + pairs.type = field_list->field[i].type; switch (pairs.type) { - case TAG_INTEGER: - pairs.value_longlong = tag_list->tag[i].value_longlong; + case FIELD_VALUE_INTEGER: + pairs.value_longlong = field_list->field[i].value_longlong; break; - case TAG_DOUBLE: - pairs.value_double = tag_list->tag[i].value_double; + case FIELD_VALUE_DOUBLE: + pairs.value_double = field_list->field[i].value_double; break; - case TAG_CSTRING: - pairs.value_str = (char *)tag_list->tag[i].value_str; + case FIELD_VALUE_CSTRING: + pairs.value_str = (char *)field_list->field[i].value_str; break; default: break; } - kv_pair_write_to_json(&pairs, cjson_tags); + kv_pair_write_to_json(&pairs, json_obj); } } -void cell_iter_read_tag_list(const struct cell_iter *iter, struct export_kv_pair **exported_tags, size_t *len_out) +void cell_iter_read_dimensions(const struct cell_iter *iter, struct export_kv_pair **exported_dimensions, size_t *n_dimension_out) { - struct fieldstat_tag_list *tag_list_cell = &iter->tag_list[iter->curr_cell_idx]; - struct fieldstat_tag_list *tag_arr_shared = iter->shared_tag; + struct field_list *field_list_cell = &iter->cell_dimension[iter->curr_cell_idx]; + struct field_list *field_list_cube = iter->cube_dimension; - size_t ret_tag_list_len = tag_list_cell->n_tag + tag_arr_shared->n_tag; + size_t ret_n_dimension = field_list_cell->n_field + field_list_cube->n_field; - struct export_kv_pair *tag_list = malloc(sizeof(struct export_kv_pair) * ret_tag_list_len); - *exported_tags = tag_list; - *len_out = ret_tag_list_len; + struct export_kv_pair *ret = malloc(sizeof(struct export_kv_pair) * ret_n_dimension); + *exported_dimensions = ret; + *n_dimension_out = ret_n_dimension; - for (int i = 0; i < tag_list_cell->n_tag; i++) { - kv_pair_fill_with_tags(tag_list++, &tag_list_cell->tag[i]); + for (int i = 0; i < field_list_cell->n_field; i++) { + kv_pair_fill_with_fields(ret++, &field_list_cell->field[i]); } - for (int i = 0; i < tag_arr_shared->n_tag; i++) { - kv_pair_fill_with_tags(tag_list++, &tag_arr_shared->tag[i]); + for (int i = 0; i < field_list_cube->n_field; i++) { + kv_pair_fill_with_fields(ret++, &field_list_cube->field[i]); } } @@ -677,7 +625,7 @@ void kv_pair_free_list(struct export_kv_pair *pairs, size_t len) { for (int i = 0; i < len; i++) { struct export_kv_pair *pair = &pairs[i]; - if (pair->type == TAG_CSTRING) { + if (pair->type == FIELD_VALUE_CSTRING) { free(pair->value_str); } free(pair->key); @@ -686,37 +634,44 @@ void kv_pair_free_list(struct export_kv_pair *pairs, size_t len) } // return 1 if added, 0 if not added -int cjson_map_add(struct cellwise_rec_for_export *tag_field_pair, const struct cell_iter *iter) +int json_obj_add(struct cellwise_rec_for_export *tag_field_pair, const struct cell_iter *iter) { struct export_kv_pair **fields = NULL; - int n_nonempty_metrics = 0; + int *metric_ids = NULL; + size_t n_metric = 0; + fieldstat_get_metric_in_cell(iter->instance, iter->cube_ids[iter->curr_cube_idx], &iter->cell_dimension[iter->curr_cell_idx], &metric_ids, &n_metric); + if (n_metric == 0) { + // printf("cannot get metric in cell\n"); + return 0; + } - for (int i = 0; i < iter->n_metric; i++) { - struct export_kv_pair *field = cell_query_with_iter(iter, iter->metric_ids[i]); + int n_nonempty_metrics = 0; + for (int i = 0; i < n_metric; i++) { + struct export_kv_pair *field = cell_query_with_iter(iter, metric_ids[i]); if (field == NULL) { continue; } if (fields == NULL) { - fields = malloc(sizeof(struct export_kv_pair *) * iter->n_metric); + fields = malloc(sizeof(struct export_kv_pair *) * n_metric); } fields[n_nonempty_metrics++] = field; } - + free(metric_ids); if (fields == NULL) { // all fields are null return 0; } tag_field_pair->metric_pairs = fields; tag_field_pair->n_metric = n_nonempty_metrics; - struct export_kv_pair *tag_list; - size_t tag_list_len; - cell_iter_read_tag_list(iter, &tag_list, &tag_list_len); + struct export_kv_pair *dimensions; + size_t n_pair; + cell_iter_read_dimensions(iter, &dimensions, &n_pair); struct json_writer *writer_tag = json_writer_init(); - for (int i = 0; i < tag_list_len; i++) { - kv_pair_write_to_json(&tag_list[i], writer_tag); + for (int i = 0; i < n_pair; i++) { + kv_pair_write_to_json(&dimensions[i], writer_tag); } - kv_pair_free_list(tag_list, tag_list_len); - tag_field_pair->cjson_tags = writer_tag; + kv_pair_free_list(dimensions, n_pair); + tag_field_pair->tag_json_obj = writer_tag; return 1; } @@ -746,7 +701,7 @@ struct cellwise_rec_for_export *read_tag_and_field(const struct fieldstat *insta int i = 0; while (cell_iter_next(iter)) { // next_num++; - if (cjson_map_add(&tag_field_pair[i], iter) == 0) { + if (json_obj_add(&tag_field_pair[i], iter) == 0) { continue; } i++; @@ -765,7 +720,7 @@ void fieldstat_json_exporter_write_global_tags(const struct fieldstat_json_expor } for (int i = 0; i < arr_len; i++) { - tag_list_append_to_tag_object(exporter->global_tag_list, tag_field_pair_arr[i].cjson_tags); + field_list_write_to_json(exporter->global_tag_list, tag_field_pair_arr[i].tag_json_obj); } } @@ -875,7 +830,7 @@ void fieldstat_json_exporter_export_array(const struct fieldstat_json_exporter * json_writer_start_map(root); json_writer_str_field(root, "name", tmp_name, strlen(tmp_name)); - json_writer_object_item(root, "tags", current->cjson_tags); + json_writer_object_item(root, "tags", current->tag_json_obj); json_writer_object_item(root, "fields", field_json); if (exporter->history != NULL) { json_writer_object_item(root, "fields_delta", field_json_delta); @@ -919,7 +874,7 @@ char *fieldstat_json_exporter_export(const struct fieldstat_json_exporter *expor int buf_len = 4096; char *cjson_str_arr = (char *)malloc(buf_len); - // cjson is so slow, so we construct the json array manually, the reason why we don't use json_writer is simply because I wrote this before implementing json_writer + // cjson is so slow, so we construct the json array manually int used_len = add_object_to_json_array_start(cjson_str_arr, buf_len); for (int i = 0; i < n_pair; i++) { used_len = add_object_to_json_array(&cjson_str_arr, &buf_len, used_len, str_arr[i]); @@ -970,7 +925,7 @@ char *fieldstat_json_exporter_export_with_delta(const struct fieldstat_json_expo json_writer_start_map(root); json_writer_str_field(root, "name", tmp_name, strlen(tmp_name)); - json_writer_object_item(root, "tags", current->cjson_tags); + json_writer_object_item(root, "tags", current->tag_json_obj); json_writer_object_item(root, "fields", field_json); json_writer_object_item(root, "fields_delta", field_json_delta); json_writer_longlong_field(root, "timestamp_ms", cal_ms_time(timestamp)); @@ -999,7 +954,7 @@ char *fieldstat_json_exporter_export_with_delta(const struct fieldstat_json_expo free(expair_delta[i].metric_pairs); char *dummy_s; size_t dummy_len; - json_writer_finish(expair_delta[i].cjson_tags, &dummy_s, &dummy_len); // the tag json is not added to the root json, so we need to free it manually + json_writer_finish(expair_delta[i].tag_json_obj, &dummy_s, &dummy_len); // the field json is not added to the root json, so we need to free it manually free(dummy_s); } free(expair_delta); @@ -1019,30 +974,30 @@ struct fieldstat_json_exporter *fieldstat_json_exporter_new() return exporter; } -void fieldstat_json_exporter_set_global_tag(struct fieldstat_json_exporter *exporter, const struct fieldstat_tag tag_list[], size_t n_tag) +void fieldstat_json_exporter_set_global_tag(struct fieldstat_json_exporter *exporter, const struct field tag_list[], size_t n_field) { if (exporter->global_tag_list != NULL) { - free(exporter->global_tag_list->tag); + free(exporter->global_tag_list->field); free(exporter->global_tag_list); } - exporter->global_tag_list = malloc(sizeof(struct fieldstat_tag_list)); - exporter->global_tag_list->n_tag = n_tag; - exporter->global_tag_list->tag = malloc(sizeof(struct fieldstat_tag) * n_tag); - for (size_t i = 0; i < n_tag; i++) { - struct fieldstat_tag *tag = &exporter->global_tag_list->tag[i]; - tag->key = strdup(tag_list[i].key); - tag->type = tag_list[i].type; - switch (tag->type) + exporter->global_tag_list = malloc(sizeof(struct field_list)); + exporter->global_tag_list->n_field = n_field; + exporter->global_tag_list->field = malloc(sizeof(struct field) * n_field); + for (size_t i = 0; i < n_field; i++) { + struct field *field = &exporter->global_tag_list->field[i]; + field->key = strdup(tag_list[i].key); + field->type = tag_list[i].type; + switch (field->type) { - case TAG_INTEGER: - tag->value_longlong = tag_list[i].value_longlong; + case FIELD_VALUE_INTEGER: + field->value_longlong = tag_list[i].value_longlong; break; - case TAG_CSTRING: - tag->value_str = strdup(tag_list[i].value_str); + case FIELD_VALUE_CSTRING: + field->value_str = strdup(tag_list[i].value_str); break; - case TAG_DOUBLE: - tag->value_double = tag_list[i].value_double; + case FIELD_VALUE_DOUBLE: + field->value_double = tag_list[i].value_double; break; default: @@ -1065,7 +1020,7 @@ void fieldstat_json_exporter_free(struct fieldstat_json_exporter *exporter) free(exporter->name); } if (exporter->global_tag_list != NULL) { - fieldstat_tag_list_arr_free(exporter->global_tag_list, 1); + fieldstat_field_list_arr_free(exporter->global_tag_list, 1); } if (exporter->history != NULL) { @@ -1082,3 +1037,81 @@ void fieldstat_json_exporter_enable_delta(struct fieldstat_json_exporter *export } exporter->history = counter_history_new(); } + +void fieldstat_json_exporter_export_flat_array(const struct fieldstat_json_exporter *exporter, const struct fieldstat *instance, const struct timeval *timestamp, char ***output, size_t *output_size) { + long long timestamp_ms = cal_ms_time(timestamp); + + size_t n_pair; + struct cellwise_rec_for_export *tag_field_pair = read_tag_and_field(instance, &n_pair); + if (tag_field_pair == NULL || n_pair == 0) { + free(tag_field_pair); // tag_field_pair is not NULL when there are registered metrics but no valid cells + *output = NULL; + *output_size = 0; + return; + } + + fieldstat_json_exporter_write_global_tags(exporter, tag_field_pair, n_pair); + + char **cjson_str_arr = (char **)malloc(sizeof(char *) * n_pair); + for (int i = 0; i < n_pair; i++) { + struct cellwise_rec_for_export *current = &tag_field_pair[i]; + struct json_writer *root = current->tag_json_obj; + for (int j = 0; j < tag_field_pair[i].n_metric; j++) { + kv_pair_write_to_json(tag_field_pair[i].metric_pairs[j], root); + } + + const char *tmp_name = exporter->name ? exporter->name : DEFAULT_EXPORTER_NAME; + json_writer_str_field(root, "name", tmp_name, strlen(tmp_name)); + json_writer_longlong_field(root, "timestamp_ms", timestamp_ms); + + char *cjson_str; + size_t cjson_str_len; + json_writer_finish(root, &cjson_str, &cjson_str_len); + char *cjson_with_braces = malloc(cjson_str_len + 3); + cjson_with_braces[0] = '{'; + memcpy(cjson_with_braces + 1, cjson_str, cjson_str_len); + cjson_with_braces[cjson_str_len + 1] = '}'; + cjson_with_braces[cjson_str_len + 2] = '\0'; + + cjson_str_arr[i] = cjson_with_braces; + free(cjson_str); + } + + *output = cjson_str_arr; + *output_size = n_pair; + + for (int i = 0; i < n_pair; i++) { + for (int j = 0; j < tag_field_pair[i].n_metric; j++) { + kv_pair_free(tag_field_pair[i].metric_pairs[j]); + } + free(tag_field_pair[i].metric_pairs); + } + + free(tag_field_pair); +} + +char *fieldstat_json_exporter_export_flat(const struct fieldstat_json_exporter *exporter, const struct fieldstat *instance, const struct timeval *timestamp) +{ + char **str_arr = NULL; + size_t n_pair = 0; + fieldstat_json_exporter_export_flat_array(exporter, instance, timestamp, &str_arr, &n_pair); + if (str_arr == NULL || n_pair == 0) { + return NULL; + } + + int buf_len = 4096; + char *cjson_str_arr = (char *)malloc(buf_len); + // cjson is so slow, so we construct the json array manually + int used_len = add_object_to_json_array_start(cjson_str_arr, buf_len); + for (int i = 0; i < n_pair; i++) { + used_len = add_object_to_json_array(&cjson_str_arr, &buf_len, used_len, str_arr[i]); + } + add_object_to_json_array_end(&cjson_str_arr, buf_len, used_len); + + for (int i = 0; i < n_pair; i++) { + free(str_arr[i]); + } + free(str_arr); + + return cjson_str_arr; +}
\ No newline at end of file diff --git a/src/exporter/fieldstat_exporter.py b/src/exporter/fieldstat_exporter.py index 49cb30f..d4e5b2c 100644 --- a/src/exporter/fieldstat_exporter.py +++ b/src/exporter/fieldstat_exporter.py @@ -25,11 +25,11 @@ class FieldstatAPI: libfieldstat.fieldstat_histogram_free.argtypes = [ctypes.c_void_p] - libfieldstat.fieldstat_histogram_value_at_percentile.argtypes = [ctypes.c_void_p, ctypes.c_double] - libfieldstat.fieldstat_histogram_value_at_percentile.restype = ctypes.c_longlong + libfieldstat.fieldstat_histogram_value_at_percentile_api.argtypes = [ctypes.c_void_p, ctypes.c_double] + libfieldstat.fieldstat_histogram_value_at_percentile_api.restype = ctypes.c_longlong - libfieldstat.fieldstat_histogram_count_le_value.argtypes = [ctypes.c_void_p, ctypes.c_longlong] - libfieldstat.fieldstat_histogram_count_le_value.restype = ctypes.c_longlong + libfieldstat.fieldstat_histogram_count_le_value_api.argtypes = [ctypes.c_void_p, ctypes.c_longlong] + libfieldstat.fieldstat_histogram_count_le_value_api.restype = ctypes.c_longlong libfieldstat.fieldstat_histogram_value_total_count.argtypes = [ctypes.c_void_p] libfieldstat.fieldstat_histogram_value_total_count.restype = ctypes.c_longlong @@ -134,7 +134,7 @@ class PrometheusExporter: metrics = "" for i in self.hist_bins: - value = FieldstatAPI.libfieldstat.fieldstat_histogram_count_le_value(c_hist, int(i)) + value = FieldstatAPI.libfieldstat.fieldstat_histogram_count_le_value_api(c_hist, int(i)) metric = name + "_bucket" + "{" + tags + ",le=\"{:.2f}\"".format(i) + "}" + ' ' + str(value) + '\n' metrics += metric self.n_lines += 1 @@ -144,7 +144,7 @@ class PrometheusExporter: def __build_summary_format(self, name, tags, c_hist): metrics = "" for i in self.hist_bins: - value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_at_percentile(c_hist, float(i * 100)) + value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_at_percentile_api(c_hist, float(i * 100)) metric = name + "{" + tags + ",quantile=\"{:.2f}%\"".format(i * 100) + "}" + ' ' + str(value) + '\n' metrics += metric self.n_lines += 1 @@ -365,10 +365,10 @@ class HistogramTable: row_values = [] for i in self.bins: if self.format == "summary": - value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_at_percentile(c_hist, float(i * 100)) + value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_at_percentile_api(c_hist, float(i * 100)) row_values.append(str(value)) if self.format == "histogram": - value = FieldstatAPI.libfieldstat.fieldstat_histogram_count_le_value(c_hist, int(i)) + value = FieldstatAPI.libfieldstat.fieldstat_histogram_count_le_value_api(c_hist, int(i)) row_values.append(str(value)) shared_values = self.__get_row_shared_values(c_hist) row_values += shared_values @@ -930,7 +930,7 @@ class FieldstatExporter: parser.add_argument("-m", "--match-tags", type = str, default = "", help = "Display the tags match metrics") parser.add_argument("-t", "--template", type = str, default = "", - help = "Specify the print template with jinja2. For specific usage, try -th") # todo，在打印错误后给一个具体的帮助信息 + help = "Specify the print template with jinja2. For specific usage, try -th") parser.add_argument("-e", "--no-tagged", action = 'store_true', default = False, help = "Print all counter metrics without tags in one list.") diff --git a/src/fieldstat.c b/src/fieldstat.c index 376bc43..bb530f1 100644 --- a/src/fieldstat.c +++ b/src/fieldstat.c @@ -6,126 +6,36 @@ #include "cjson/cJSON.h" #include "uthash.h" -#include "serializer.h" + #include "fieldstat.h" #include "metrics/metric.h" -#include "tags/cell_manager.h" - -#define DEFAULT_N_METRIC 64 -#define DEFAULT_N_CUBE 128 - -struct fs_cube { - enum sampling_mode sampling_mode; - struct cell_manager *cell_manager; - size_t max_n_cell; - int next_cell_id; - int primary_metric_id; - - // the key of cube is the combination of shared tags - struct fieldstat_tag *shared_tags; - size_t n_shared_tags; - struct tag_hash_key *key_tag; - - struct metric **metrics; - size_t valid_metric_arr_len; - size_t max_n_metric; -}; - -struct metric_name_id_map { - char *name; - int id; - UT_hash_handle hh; -}; +#include "cube.h" struct fieldstat { - struct fs_cube **cube; - unsigned long *cube_version; // increase from 0 every time the cube is deleted - unsigned long cell_version; // increase from 0 every time fieldstat_reset is called - size_t valid_cube_arr_length; - size_t max_n_cube; - - struct metric **metric_masters; - size_t n_metric_master; - size_t max_n_metric_master; - struct metric_name_id_map *metric_name_id_map; - - struct cube_manager *shared_tag_cube_manager; + struct cube_manager *cube_manager; }; -int name_id_map_get_id_by_name(struct metric_name_id_map *map, const char *metric_name) -{ - struct metric_name_id_map *entry = NULL; - HASH_FIND_STR(map, metric_name, entry); - if (entry == NULL) { - return -1; - } - return entry->id; -} - -void name_id_map_add(struct metric_name_id_map **map, const char *name, int id) -{ - struct metric_name_id_map *entry = malloc(sizeof(struct metric_name_id_map)); - entry->id = id; - entry->name = strdup(name); - HASH_ADD_KEYPTR(hh, *map, entry->name, strlen(entry->name), entry); -} - -void name_id_map_free(struct metric_name_id_map *map) -{ - struct metric_name_id_map *entry, *tmp; - HASH_ITER(hh, map, entry, tmp) { - HASH_DEL(map, entry); - free(entry->name); - free(entry); - } -} - -struct metric_name_id_map *name_id_map_copy(struct metric_name_id_map *map) -{ - struct metric_name_id_map *map_dup = NULL; - struct metric_name_id_map *entry, *tmp; - HASH_ITER(hh, map, entry, tmp) { - name_id_map_add(&map_dup, entry->name, entry->id); - } - return map_dup; -} +/* -------------------------------------------------------------------------- */ +/* fieldstat */ +/* -------------------------------------------------------------------------- */ struct fieldstat *fieldstat_new() { struct fieldstat *instance = calloc(1, sizeof(struct fieldstat)); - instance->max_n_cube = DEFAULT_N_CUBE; - instance->cube = calloc(instance->max_n_cube, sizeof(struct fs_cube *)); - instance->cube_version = calloc(instance->max_n_cube, sizeof(unsigned long)); - - instance->max_n_metric_master = DEFAULT_N_METRIC; - instance->metric_masters = calloc(instance->max_n_metric_master, sizeof(struct metric *)); - instance->metric_name_id_map = NULL; - - instance->shared_tag_cube_manager = cube_manager_new(); + instance->cube_manager = cube_manager_new(); return instance; } -void fieldstat_cube_free(struct fieldstat *instance, int cube_id); void fieldstat_free(struct fieldstat *instance) { if (instance == NULL) { return; } - for (size_t i = 0; i < instance->valid_cube_arr_length; i++) { - fieldstat_cube_free(instance, i); - } - free(instance->cube); - free(instance->cube_version); - cube_manager_free(instance->shared_tag_cube_manager); - for (size_t i = 0; i < instance->n_metric_master; i++) { - metric_free(instance->metric_masters[i]); - } - free(instance->metric_masters); - name_id_map_free(instance->metric_name_id_map); + cube_manager_free(instance->cube_manager); free(instance); } @@ -135,772 +45,163 @@ void fieldstat_reset(struct fieldstat *instance) if (instance == NULL) { return; } - for (size_t i = 0; i < instance->valid_cube_arr_length; i++) { - struct fs_cube *cube = instance->cube[i]; - if (cube == NULL) { - continue; - } - for (size_t j = 0; j < cube->valid_metric_arr_len; j++) { - if (cube->metrics[j] == NULL) { - continue; - } - metric_reset(cube->metrics[j]); - } - - cell_manager_reset(cube->cell_manager); - } - instance->cell_version++; -} - -unsigned long fieldstat_get_version(const struct fieldstat *instance) -{ - if (instance == NULL) { - return 0; - } - return instance->cell_version; + cube_manager_reset(instance->cube_manager); } -int fieldstat_destroy_cube(struct fieldstat *instance, int cube_id) +int fieldstat_cube_destroy(struct fieldstat *instance, int cube_id) { - if (instance == NULL) { - return FS_ERR_NULL_HANDLER; - } - if (cube_id < 0 || cube_id >= instance->valid_cube_arr_length) { - return FS_ERR_INVALID_CUBE_ID; - } - if (instance->cube[cube_id] == NULL) { + struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + if (cube == NULL) { return FS_ERR_INVALID_CUBE_ID; } - const struct fs_cube *cube = instance->cube[cube_id]; - cube_manager_delete(instance->shared_tag_cube_manager, cube->key_tag); - - fieldstat_cube_free(instance, cube_id); - - instance->cube[cube_id] = NULL; - instance->cube_version[cube_id]++; - return 0; -} - -long long fieldstat_get_cube_version(const struct fieldstat *instance, int cube_id) -{ - if (instance == NULL) { - return FS_ERR_NULL_HANDLER; - } - if (cube_id < 0 || cube_id >= instance->valid_cube_arr_length) { - return FS_ERR_INVALID_CUBE_ID; - } - if (instance->cube[cube_id] == NULL && instance->cube_version[cube_id] == 0) { - return FS_ERR_INVALID_CUBE_ID; - } + cube_manager_delete(instance->cube_manager, cube); - return instance->cube_version[cube_id]; + return FS_OK; } /* -------------------------------------------------------------------------- */ /* cube */ /* -------------------------------------------------------------------------- */ -void fieldstat_free_tag_array(struct fieldstat_tag *tags, size_t n_tags) -{ - for (size_t i = 0; i < n_tags; i++) { - struct fieldstat_tag *tag = &tags[i]; - free((char *)tag->key); - if (tag->type == TAG_CSTRING) { - free((char *)tag->value_str); - } - } - free(tags); -} - -void add_cube_to_position(struct fieldstat *instance, struct fs_cube *cube, int cube_id) -{ - if (cube_id >= instance->max_n_cube) { - instance->max_n_cube *= 2; - struct fs_cube **old_cube_arr = instance->cube; - instance->cube = calloc(instance->max_n_cube, sizeof(struct fs_cube *)); - memcpy(instance->cube, old_cube_arr, sizeof(struct fs_cube *) * instance->valid_cube_arr_length); - free(old_cube_arr); - - unsigned long *old_ver_arr = instance->cube_version; - instance->cube_version = calloc(instance->max_n_cube, sizeof(unsigned long)); - memcpy(instance->cube_version, old_ver_arr, sizeof(unsigned long) * instance->valid_cube_arr_length); - free(old_ver_arr); - } - instance->cube[cube_id] = cube; - if (cube_id >= instance->valid_cube_arr_length) { - instance->valid_cube_arr_length = cube_id + 1; - } -} - -int fieldstat_append_cube_to_instance(struct fieldstat *instance, struct fs_cube *cube) -{ - for (int i = 0; i < instance->valid_cube_arr_length; i++) { - if (instance->cube[i] == NULL) { - instance->cube[i] = cube; - cube_manager_add(instance->shared_tag_cube_manager, cube->key_tag, i); - return i; +// cppcheck-suppress [constParameterPointer, unmatchedSuppression] +int fieldstat_cube_set_sampling(struct fieldstat *instance, int cube_id, enum sampling_mode mode, int max_n_cell, int primary_metric_id) { + if (max_n_cell <= 0) { + if (mode != SAMPLING_MODE_COMPREHENSIVE) { + return FS_ERR_MAX_N_CELL_LESS_THAN_ZERO; + } else { + max_n_cell = INT32_MAX; } } - int cube_id = instance->valid_cube_arr_length; - add_cube_to_position(instance, cube, cube_id); - cube_manager_add(instance->shared_tag_cube_manager, cube->key_tag, cube_id); - - return cube_id; -} -struct fs_cube *fieldstat_cube_info_init(const struct fieldstat_tag *shared_tags, size_t n_tag, enum sampling_mode mode, size_t max_n_cell) -{ - struct fs_cube *cube = calloc(1, sizeof(struct fs_cube)); - cube->sampling_mode = mode; - cube->max_n_cell = max_n_cell; - if (n_tag == 0) { - cube->shared_tags = NULL; - } else { - cube->shared_tags = malloc(sizeof(struct fieldstat_tag) * n_tag); - for (int i = 0; i < n_tag; i++) { - struct fieldstat_tag *dest_tag = &cube->shared_tags[i]; - dest_tag->key = strdup(shared_tags[i].key); - dest_tag->type = shared_tags[i].type; - switch (dest_tag->type) - { - case TAG_INTEGER: - dest_tag->value_longlong = shared_tags[i].value_longlong; - break; - case TAG_CSTRING: - dest_tag->value_str = strdup(shared_tags[i].value_str); - break; - case TAG_DOUBLE: - dest_tag->value_double = shared_tags[i].value_double; - break; - - default: - break; - } - } + struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + if (cube == NULL) { + return FS_ERR_INVALID_CUBE_ID; } - - cube->n_shared_tags = n_tag; - - struct tag_hash_key *shared_tag_key = malloc(sizeof(struct tag_hash_key)); - tag_hash_key_init_with_fieldstat_tag(shared_tag_key, shared_tags, n_tag, true); - cube->key_tag = shared_tag_key; - - cube->max_n_metric = 64; - cube->metrics = calloc(cube->max_n_metric, sizeof(struct metric *)); - - return cube; -} - -struct fs_cube *fieldstat_cube_new(const struct fieldstat_tag *shared_tags, size_t n_tag, enum sampling_mode mode, size_t max_n_cell) -{ - struct fs_cube *cube = fieldstat_cube_info_init(shared_tags, n_tag, mode, max_n_cell); - - cube->cell_manager = cell_manager_new(mode, max_n_cell); - - return cube; + + return cube_set_sampling(cube, mode, max_n_cell, primary_metric_id); } -int fieldstat_create_cube(struct fieldstat *instance, const struct fieldstat_tag *shared_tags, size_t n_tag, enum sampling_mode mode, size_t max_n_cell) +int fieldstat_cube_create(struct fieldstat *instance, const struct field *cube_dimensions, size_t n_dimension) { if (instance == NULL) { return FS_ERR_NULL_HANDLER; } - if (n_tag == 0 || shared_tags == NULL) { - shared_tags = NULL; - n_tag = 0; - } - if (mode == SAMPLING_MODE_TOPK && max_n_cell == 0) { - return FS_ERR_INVALID_PARAM; - } - if (max_n_cell == 0) { - max_n_cell = INT32_MAX; - } - struct tag_hash_key shared_tag_key; - tag_hash_key_init_with_fieldstat_tag(&shared_tag_key, shared_tags, n_tag, false); - int ret = cube_manager_find(instance->shared_tag_cube_manager, &shared_tag_key); - if (ret != -1) { - return FS_ERR_INVALID_KEY; - } - - struct fs_cube *cube = fieldstat_cube_new(shared_tags, n_tag, mode, max_n_cell); - - int cube_id = fieldstat_append_cube_to_instance(instance, cube); - - return cube_id; -} - -int fieldstat_cube_add(struct fieldstat *instance, int cube_id, const struct fieldstat_tag *tags, size_t n_tag, long long occurrence) -{ - struct fs_cube *cube = instance->cube[cube_id]; - - int ret = -1; - struct tag_hash_key tag_key; - - tag_hash_key_init_with_fieldstat_tag(&tag_key, tags, n_tag, false); - if (cube->sampling_mode == SAMPLING_MODE_COMPREHENSIVE) { - ret = cell_manager_add_cell(cube->cell_manager, &tag_key); - } else { - int popped_cell_id = -1; - ret = cell_manager_add_cell_topk(cube->cell_manager, &tag_key, occurrence, &popped_cell_id); - - if (popped_cell_id != -1) { - for (size_t i = 0; i < cube->valid_metric_arr_len; i++) { - if (cube->metrics[i] == NULL) { - continue; - } - metric_delete_cell(cube->metrics[i], popped_cell_id); - } - } + if (n_dimension == 0 || cube_dimensions == NULL) { + cube_dimensions = NULL; + n_dimension = 0; } + struct cube *cube = cube_new(cube_dimensions, n_dimension); + int ret = cube_manager_add(instance->cube_manager, cube); if (ret < 0) { - return -1; - } - return ret; -} - -void fieldstat_cube_free_contents(struct fieldstat *instance, int cube_id) -{ - struct fs_cube *cube = instance->cube[cube_id]; - - cell_manager_free(cube->cell_manager); - fieldstat_free_tag_array(cube->shared_tags, cube->n_shared_tags); - tag_hash_key_free(cube->key_tag); - - for (size_t i = 0; i < cube->valid_metric_arr_len; i++) { - if (cube->metrics[i] != NULL) { - metric_free(cube->metrics[i]); - } - } - free(cube->metrics); - - free(cube); - instance->cube[cube_id] = NULL; - - if (cube_id == instance->valid_cube_arr_length - 1) { - instance->valid_cube_arr_length--; - } -} - -void fieldstat_cube_free(struct fieldstat *instance, int cube_id) -{ - if (instance->cube[cube_id] == NULL) { - return; - } - - cube_manager_delete(instance->shared_tag_cube_manager, instance->cube[cube_id]->key_tag); - - fieldstat_cube_free_contents(instance, cube_id); -} - -struct fs_cube *fieldstat_cube_fork(const struct fs_cube *cube) { - struct fs_cube *ret = fieldstat_cube_new(cube->shared_tags, cube->n_shared_tags, cube->sampling_mode, cube->max_n_cell); - ret->primary_metric_id = cube->primary_metric_id; - - return ret; -} - -int fieldstat_cube_set_primary_metric(struct fieldstat *instance, int cube_id, int metric_id) -{ - if (instance == NULL) { - return FS_ERR_NULL_HANDLER; - } - if (cube_id < 0 || cube_id >= instance->valid_cube_arr_length) { - return FS_ERR_INVALID_CUBE_ID; + cube_free(cube); + return FS_ERR_DIMENSION_ALREADY_EXISTS; } - struct fs_cube *cube = instance->cube[cube_id]; - if (cube == NULL) { - return FS_ERR_INVALID_CUBE_ID; - } - if (cube->sampling_mode != SAMPLING_MODE_TOPK) { - return FS_ERR_INVALID_PARAM; - } - if (metric_id < 0 || metric_id >= instance->n_metric_master) { - return FS_ERR_INVALID_METRIC_ID; - } - if (instance->metric_masters[metric_id] == NULL) { - return FS_ERR_INVALID_METRIC_ID; - } - if (metric_get_type(instance->metric_masters[metric_id]) != METRIC_TYPE_COUNTER) { - return FS_ERR_INVALID_PARAM; - } - - cube->primary_metric_id = metric_id; - return FS_OK; + return ret; //ret is the cube_id } /* -------------------------------------------------------------------------- */ /* metric register */ /* -------------------------------------------------------------------------- */ - -void add_metric_to_cube(struct fs_cube *cube, struct metric *metric, int metric_id) -{ - if (metric_id >= cube->max_n_metric) { - cube->max_n_metric *= 2; - cube->metrics = realloc(cube->metrics, sizeof(struct metric *) * cube->max_n_metric); - memset(cube->metrics + cube->valid_metric_arr_len, 0, sizeof(struct metric *) * (cube->max_n_metric - cube->valid_metric_arr_len)); - } - - cube->metrics[metric_id] = metric; - if (metric_id >= cube->valid_metric_arr_len) { - cube->valid_metric_arr_len = metric_id + 1; - } -} - -void add_metric_to_instance(struct fieldstat *instance, const struct metric *metric, int metric_id) -{ - if (metric_id >= instance->max_n_metric_master) { - instance->max_n_metric_master *= 2; - instance->metric_masters = realloc(instance->metric_masters, sizeof(struct metric *) * instance->max_n_metric_master); - memset(instance->metric_masters + instance->n_metric_master, 0, sizeof(struct metric *) * (instance->max_n_metric_master - instance->n_metric_master)); - } - - instance->metric_masters[metric_id] = (struct metric *)metric; - if (metric_id >= instance->n_metric_master) { - instance->n_metric_master = metric_id + 1; - } - name_id_map_add(&instance->metric_name_id_map, metric_get_name(metric), metric_id); -} - -struct metric *find_or_add_metric(struct fieldstat *instance, int cube_id, int metric_id) { - struct fs_cube *cube = instance->cube[cube_id]; - if (metric_id < cube->valid_metric_arr_len && cube->metrics[metric_id] != NULL) { - return cube->metrics[metric_id]; - } - struct metric *metric = metric_fork(instance->metric_masters[metric_id]); - add_metric_to_cube(cube, metric, metric_id); - return metric; -} - -static int append_metric_to_instance(struct fieldstat *instance, const struct metric *metric) -{ - int metric_id = instance->n_metric_master; - add_metric_to_instance(instance, metric, metric_id); - - return metric_id; -} - -int check_before_register_metric(const struct fieldstat *instance, const char *metric_name) -{ - if (instance == NULL) { - return FS_ERR_NULL_HANDLER; - } - - if (name_id_map_get_id_by_name(instance->metric_name_id_map, metric_name) != -1) { - return FS_ERR_INVALID_KEY; - } - - return FS_OK; -} - -int fieldstat_register_counter(struct fieldstat *instance, const char *metric_name) +// cppcheck-suppress [constParameterPointer, unmatchedSuppression] +int fieldstat_register_counter(struct fieldstat *instance, int cube_id, const char *metric_name) { - int ret = check_before_register_metric(instance, metric_name); - if (ret != FS_OK) { - return ret; + struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + if (cube == NULL) { + return FS_ERR_INVALID_CUBE_ID; } - - const struct metric *metric = metric_counter_new(metric_name); - - return append_metric_to_instance(instance, metric); + return cube_register_counter(cube, metric_name); } -int fieldstat_register_hll(struct fieldstat *instance, const char *metric_name, unsigned char precision) +// cppcheck-suppress [constParameterPointer, unmatchedSuppression] +int fieldstat_register_hll(struct fieldstat *instance, int cube_id, const char *metric_name, unsigned char precision) { - int ret = check_before_register_metric(instance, metric_name); - if (ret != FS_OK) { - return ret; - } - if (precision < 4 || precision > 18) { - return FS_ERR_INVALID_PARAM; + struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + if (cube == NULL) { + return FS_ERR_INVALID_CUBE_ID; } - - const struct metric *metric = metric_hll_new(metric_name, precision); - - return append_metric_to_instance(instance, metric); + return cube_register_hll(cube, metric_name, precision); } -int fieldstat_register_hist(struct fieldstat *instance, const char *metric_name, long long lowest_trackable_value, long long highest_trackable_value, int significant_figures) +// cppcheck-suppress [constParameterPointer, unmatchedSuppression] +int fieldstat_register_histogram(struct fieldstat *instance, int cube_id, const char *metric_name, long long lowest_trackable_value, long long highest_trackable_value, int significant_figures) { - int ret = check_before_register_metric(instance, metric_name); - if (ret != FS_OK) { - return ret; - } - // refer to hdr_histogram.h for the rules of parameters. Just copy them here - if (lowest_trackable_value < 1) { - return FS_ERR_INVALID_PARAM; - } - if (significant_figures < 1 || significant_figures > 5) { - return FS_ERR_INVALID_PARAM; + struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + if (cube == NULL) { + return FS_ERR_INVALID_CUBE_ID; } - if (lowest_trackable_value * 2 > highest_trackable_value) - { - return FS_ERR_INVALID_PARAM; - } - - const struct metric *metric = metric_histogram_new(metric_name, lowest_trackable_value, highest_trackable_value, significant_figures); - - return append_metric_to_instance(instance, metric); + return cube_register_hist(cube, metric_name, lowest_trackable_value, highest_trackable_value, significant_figures); } /* -------------------------------------------------------------------------- */ /* metric operation */ /* -------------------------------------------------------------------------- */ -static struct metric *fieldstat_find_metric(const struct fieldstat *instance, int cube_id, int metric_id, int *err_code) +// cppcheck-suppress [constParameterPointer, unmatchedSuppression] +int fieldstat_counter_incrby(struct fieldstat *instance, int cube_id, int metric_id, const struct field *cell_dimensions, size_t n_dimensions, long long increment) { - if (instance == NULL) { - *err_code = FS_ERR_NULL_HANDLER; - return NULL; - } - if (cube_id < 0 || cube_id >= instance->valid_cube_arr_length) { - *err_code = FS_ERR_INVALID_CUBE_ID; - return NULL; - } - struct fs_cube *cube = instance->cube[cube_id]; + struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); if (cube == NULL) { - *err_code = FS_ERR_INVALID_CUBE_ID; - return NULL; - } - - *err_code = FS_OK; - return cube->metrics[metric_id]; -} - -int check_before_add(const struct fieldstat *instance, int cube_id, int metric_id, enum metric_type type) -{ - if (instance == NULL) { - return FS_ERR_NULL_HANDLER; - } - - if (cube_id < 0 || cube_id >= instance->valid_cube_arr_length) { return FS_ERR_INVALID_CUBE_ID; } - if (instance->cube[cube_id] == NULL) { - return FS_ERR_INVALID_CUBE_ID; - } - - if (metric_id < 0 || metric_id >= instance->n_metric_master) { - return FS_ERR_INVALID_METRIC_ID; - } - const struct metric *metric = instance->metric_masters[metric_id]; - if (metric == NULL || metric_get_type(metric) != type) { - return FS_ERR_INVALID_METRIC_ID; - } - - return FS_OK; -} - -int fieldstat_counter_incrby(struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag *tags, size_t n_tag, long long increment) -{ - int ret = check_before_add(instance, cube_id, metric_id, METRIC_TYPE_COUNTER); - if (ret != FS_OK) { - return ret; - } - const struct fs_cube *cube = instance->cube[cube_id]; - long long occurrence = 0; - if (cube->sampling_mode == SAMPLING_MODE_TOPK && cube->primary_metric_id == metric_id) { - if (increment < 0) { - return FS_ERR_INVALID_PARAM; - } - - occurrence = increment; - } - - int cell_id = fieldstat_cube_add(instance, cube_id, tags, n_tag, occurrence); - if (cell_id < 0) { - return FS_ERR_TOO_MANY_CELLS; - } - - struct metric *metric = find_or_add_metric(instance, cube_id, metric_id); - - metric_counter_incrby(metric, cell_id, increment); - return FS_OK; -} - -int fieldstat_counter_incrby_batch(struct fieldstat *instance, int cube_id, int metric_ids[], const struct fieldstat_tag *tags, size_t n_tag, long long increments[], int n_metric) -{ - int ret = check_before_add(instance, cube_id, metric_ids[0], METRIC_TYPE_COUNTER); - if (ret != FS_OK) { - return ret; - } - const struct fs_cube *cube = instance->cube[cube_id]; - long long occurrence = 0; - if (cube->sampling_mode == SAMPLING_MODE_TOPK) { - for (int i = 0; i < n_metric; i++) { - if (metric_ids[i] == cube->primary_metric_id) { - if (increments[i] < 0) { - return FS_ERR_INVALID_PARAM; - } - occurrence += increments[i]; - } - } - } - int cell_id = fieldstat_cube_add(instance, cube_id, tags, n_tag, occurrence); - if (cell_id < 0) { - return FS_ERR_TOO_MANY_CELLS; - } - - for (int i = 0; i < n_metric; i++) { - struct metric *metric = find_or_add_metric(instance, cube_id, metric_ids[i]); - metric_counter_incrby(metric, cell_id, increments[i]); - } - return FS_OK; -} - -int fieldstat_counter_set(struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag *tags, size_t n_tag, long long value) -{ - int ret = check_before_add(instance, cube_id, metric_id, METRIC_TYPE_COUNTER); - if (ret != FS_OK) { - return ret; - } - const struct fs_cube *cube = instance->cube[cube_id]; - struct metric *metric = find_or_add_metric(instance, cube_id, metric_id); - int cell_id = -1; - - // get the occurrence - long long occurrence = 0; - if (cube->sampling_mode == SAMPLING_MODE_TOPK && cube->primary_metric_id == metric_id) { - struct tag_hash_key tag_key; - tag_hash_key_init_with_fieldstat_tag(&tag_key, tags, n_tag, false); - cell_id = cell_manager_find(cube->cell_manager, &tag_key); - if (cell_id >= 0) { - long long old_value; - int exist_flag = metric_counter_get(metric, cell_id, &old_value); - if (exist_flag == 0) { // cell already exist in metric - occurrence = value - old_value; - } else { - occurrence = value; - } - } else { - occurrence = value; - } - if (occurrence < 0) { - return FS_ERR_INVALID_PARAM; - } - } - - cell_id = fieldstat_cube_add(instance, cube_id, tags, n_tag, occurrence); - if (cell_id < 0) { - return FS_ERR_TOO_MANY_CELLS; - } - metric_counter_set(metric, cell_id, value); - return 0; + return cube_counter_incrby(cube, metric_id, cell_dimensions, n_dimensions, increment); } -int fieldstat_hll_add(struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag *tags, size_t n_tag, const char *key, size_t key_len) +// cppcheck-suppress [constParameterPointer, unmatchedSuppression] +int fieldstat_counter_set(struct fieldstat *instance, int cube_id, int metric_id, const struct field *cell_dimensions, size_t n_dimensions, long long value) { - int ret = check_before_add(instance, cube_id, metric_id, METRIC_TYPE_HLL); - if (ret != FS_OK) { - return ret; - } - if (instance->cube[cube_id]->sampling_mode == SAMPLING_MODE_TOPK) { - return FS_ERR_INVALID_PARAM; - } - int cell_id = fieldstat_cube_add(instance, cube_id, tags, n_tag, 0); - if (cell_id < 0) { - return FS_ERR_TOO_MANY_CELLS; - } - struct metric *metric = find_or_add_metric(instance, cube_id, metric_id); - - metric_hll_add(metric, cell_id, key, key_len); - return 0; -} - -int fieldstat_hist_record(struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag *tags, size_t n_tag, long long value) -{ - int ret = check_before_add(instance, cube_id, metric_id, METRIC_TYPE_HISTOGRAM); - if (ret != FS_OK) { - return ret; - } - if (instance->cube[cube_id]->sampling_mode == SAMPLING_MODE_TOPK) { - return FS_ERR_INVALID_PARAM; - } - - int cell_id = fieldstat_cube_add(instance, cube_id, tags, n_tag, 0); - if (cell_id < 0) { - return FS_ERR_TOO_MANY_CELLS; + struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + if (cube == NULL) { + return FS_ERR_INVALID_CUBE_ID; } - struct metric *metric = find_or_add_metric(instance, cube_id, metric_id); - ret = metric_histogram_record(metric, cell_id, value); - if (ret < 0) { // it's ok not to recover the cell and metrics even if they are new ones, since unused ones will not be exported. - return FS_ERR_INVALID_PARAM; - } - return 0; + return cube_counter_set(cube, metric_id, cell_dimensions, n_dimensions, value); } - -/* -------------------------------------------------------------------------- */ -/* merge */ -/* -------------------------------------------------------------------------- */ - -struct fs_cube *fieldstat_cube_dup(const struct fs_cube *cube, const int *metric_id_map) +// cppcheck-suppress [constParameterPointer, unmatchedSuppression] +int fieldstat_hll_add(struct fieldstat *instance, int cube_id, int metric_id, const struct field *cell_dimensions, size_t n_dimensions, const char *key, size_t key_len) { - struct fs_cube *cube_dup = fieldstat_cube_info_init(cube->shared_tags, cube->n_shared_tags, cube->sampling_mode, cube->max_n_cell); - const struct cell_manager *cm_src = cube->cell_manager; - - struct cell_manager *cm_dup = cell_manager_copy(cm_src); - cube_dup->cell_manager = cm_dup; - cube_dup->primary_metric_id = metric_id_map[cube->primary_metric_id]; - - for (int i = 0; i < cube->valid_metric_arr_len; i ++) { - if (cube->metrics[i] == NULL) { - continue; - } - struct metric *metric_dup = metric_copy(cube->metrics[i]); - add_metric_to_cube(cube_dup, metric_dup, metric_id_map[i]); + struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + if (cube == NULL) { + return FS_ERR_INVALID_CUBE_ID; } - return cube_dup; -} - -void fieldstat_cube_merge_comprehensive(struct fs_cube *dest, const struct fs_cube *src, const int *metric_id_src_dest_map) -{ - const struct cell_manager *cell_manager_src = src->cell_manager; - struct cell_manager *cell_manager_dest = dest->cell_manager; - - int arr_len_src = 0; - const struct tag_hash_key **tag_arr_src = cell_manager_dump(cell_manager_src, &arr_len_src); - for (int cell_id_src = 0; cell_id_src < arr_len_src; cell_id_src++) { - const struct tag_hash_key *tag_src = tag_arr_src[cell_id_src]; - if (tag_src == NULL) { - continue; - } - int cell_id_final = cell_manager_add_cell(cell_manager_dest, tag_src); - if (cell_id_final == -1) { // dest is full - break; - } - for (int metric_id_src = 0; metric_id_src < src->valid_metric_arr_len; metric_id_src++) { - if (src->metrics[metric_id_src] == NULL) { - continue; - } - int metric_id_dest = metric_id_src_dest_map[metric_id_src]; - (void)metric_merge_or_copy_cell(dest->metrics[metric_id_dest], src->metrics[metric_id_src], cell_id_final, cell_id_src); - } - } + return cube_hll_add(cube, metric_id, cell_dimensions, n_dimensions, key, key_len); } -void fieldstat_cube_merge_topk(struct fs_cube *dest, const struct fs_cube *src, const int *metric_id_src_dest_map) +// cppcheck-suppress [constParameterPointer, unmatchedSuppression] +int fieldstat_hll_add_field(struct fieldstat *instance, int cube_id, int metric_id, const struct field *cell_dimensions, size_t n_dimensions, const struct field *item, size_t item_len) { - const struct cell_manager *cell_manager_src = src->cell_manager; - struct cell_manager *cell_manager_dest = dest->cell_manager; - int *cell_id_added = NULL; - int *cell_id_old = NULL; - int n_cell_id_added = 0; - int *cell_id_popped = NULL; - int n_cell_id_popped = 0; - - cell_manager_merge_topk(cell_manager_dest, cell_manager_src, &cell_id_popped, &n_cell_id_popped, &cell_id_old, &cell_id_added, &n_cell_id_added); - - for (int i = 0; i < n_cell_id_added; i++) { - int tmp_id_dest = cell_id_added[i]; - int tmp_id_src = cell_id_old[i]; - - for (int j = 0; j < src->valid_metric_arr_len; j++) { - if (src->metrics[j] == NULL) { - continue; - } - int metric_id_dest = metric_id_src_dest_map[j]; - metric_merge_or_copy_cell(dest->metrics[metric_id_dest], src->metrics[j], tmp_id_dest, tmp_id_src); - } - } - // chances are that: a cell exists in both dest and src, but it is not in final. In this case, these cells are both in cell_id_added and cell_id_popped. - // Since all cells are counter, which is easy to merge, we just delete these cells after merging them. - for (int i = 0; i < n_cell_id_popped;i++) { - int id = cell_id_popped[i]; - - for (int j = 0; j < dest->valid_metric_arr_len; j++) { - if (dest->metrics[j] == NULL) { - continue; - } - metric_delete_cell(dest->metrics[j], id); - } + struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + if (cube == NULL) { + return FS_ERR_INVALID_CUBE_ID; } - free(cell_id_popped); - free(cell_id_added); - free(cell_id_old); + return cube_hll_add_field(cube, metric_id, cell_dimensions, n_dimensions, item, item_len); } -void fieldstat_cube_merge(struct fs_cube *dest, const struct fs_cube *src, const int *metric_id_src_dest_map) +// cppcheck-suppress [constParameterPointer, unmatchedSuppression] +int fieldstat_histogram_record(struct fieldstat *instance, int cube_id, int metric_id, const struct field *cell_dimensions, size_t n_dimensions, long long value) { - for (int metric_id_src = 0; metric_id_src < src->valid_metric_arr_len; metric_id_src++) { - if (src->metrics[metric_id_src] == NULL) { - continue; - } - - int metric_id_dest = metric_id_src_dest_map[metric_id_src]; - if (dest->metrics[metric_id_dest] != NULL) { - continue; - } - - struct metric *metric_dest = metric_fork(src->metrics[metric_id_src]); - add_metric_to_cube(dest, metric_dest, metric_id_dest); + struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + if (cube == NULL) { + return FS_ERR_INVALID_CUBE_ID; } - if (dest->sampling_mode == SAMPLING_MODE_COMPREHENSIVE) { - return fieldstat_cube_merge_comprehensive(dest, src, metric_id_src_dest_map); - } - fieldstat_cube_merge_topk(dest, src, metric_id_src_dest_map); + return cube_histogram_record(cube, metric_id, cell_dimensions, n_dimensions, value); } -int fieldstat_merge(struct fieldstat *instance, struct fieldstat *src) +int fieldstat_merge(struct fieldstat *instance, const struct fieldstat *src) { if (instance == NULL || src == NULL) { return FS_ERR_NULL_HANDLER; } - int metric_id_src_dest_map[src->n_metric_master]; // every metric <src_id> in src move to metric_id_src_dest_map[<src_id>] in dst instance - for (int metric_id_src = 0; metric_id_src < src->n_metric_master; metric_id_src++) { - const char *name_src = metric_get_name(src->metric_masters[metric_id_src]); - int metric_id_dst = name_id_map_get_id_by_name(instance->metric_name_id_map, name_src); - if (metric_id_dst == -1) { - metric_id_dst = append_metric_to_instance(instance, metric_fork(src->metric_masters[metric_id_src])); - name_id_map_add(&instance->metric_name_id_map, name_src, metric_id_dst); - } - metric_id_src_dest_map[metric_id_src] = metric_id_dst; - } - - size_t n_cube_src = src->valid_cube_arr_length; - const struct cube_manager *tag_cube_id_map = instance->shared_tag_cube_manager; - int ret = 0; - for (int i = 0; i < n_cube_src; i++) { - const struct fs_cube *cube_src = src->cube[i]; - if (cube_src == NULL) { - continue; - } - - const struct tag_hash_key *shared_tag_key_src = cube_src->key_tag; - int cube_id_tmp = cube_manager_find(tag_cube_id_map, shared_tag_key_src); - if (cube_id_tmp == -1) { - struct fs_cube *copied_cube = fieldstat_cube_dup(cube_src, metric_id_src_dest_map); - fieldstat_append_cube_to_instance(instance, copied_cube); - } else { - struct fs_cube *cube_dst = instance->cube[cube_id_tmp]; - if (cube_dst->sampling_mode != cube_src->sampling_mode) { - ret = FS_ERR_INVALID_PARAM; - continue; - } - if (cube_dst->sampling_mode == SAMPLING_MODE_TOPK && strcmp( - metric_get_name(instance->metric_masters[cube_dst->primary_metric_id]), - metric_get_name(src->metric_masters[cube_src->primary_metric_id]) - ) != 0) { - printf("primary metric name not match, name instance: %s, name src: %s\n", - metric_get_name(instance->metric_masters[cube_dst->primary_metric_id]), - metric_get_name(src->metric_masters[cube_src->primary_metric_id])); - ret = FS_ERR_INVALID_PARAM; - continue; - } - fieldstat_cube_merge(cube_dst, cube_src, metric_id_src_dest_map); - } - } - - return ret; + return cube_manager_merge(instance->cube_manager, src->cube_manager); } struct fieldstat *fieldstat_fork(const struct fieldstat *instance) @@ -909,132 +210,14 @@ struct fieldstat *fieldstat_fork(const struct fieldstat *instance) return NULL; } struct fieldstat *new_instance = calloc(1, sizeof(struct fieldstat)); - new_instance->shared_tag_cube_manager = cube_manager_new(); - - new_instance->valid_cube_arr_length = instance->valid_cube_arr_length; - new_instance->max_n_cube = instance->max_n_cube; - new_instance->cube = calloc(new_instance->max_n_cube, sizeof(struct fs_cube *)); - for (size_t i = 0; i < new_instance->valid_cube_arr_length; i++) { - struct fs_cube *cube = instance->cube[i]; - if (cube == NULL) { - continue; - } - new_instance->cube[i] = fieldstat_cube_fork(cube); - cube_manager_add(new_instance->shared_tag_cube_manager, cube->key_tag, i); - // copy registered metrics - for (size_t j = 0; j < cube->valid_metric_arr_len; j++) { - const struct metric *metric = cube->metrics[j]; - if (metric == NULL) { - continue; - } - struct metric *new_metric = metric_fork(metric); - add_metric_to_cube(new_instance->cube[i], new_metric, j); - } - } - new_instance->cube_version = calloc(new_instance->max_n_cube, sizeof(unsigned long)); - memcpy(new_instance->cube_version, instance->cube_version, sizeof(unsigned long) * new_instance->max_n_cube); - - new_instance->metric_masters = calloc(instance->max_n_metric_master, sizeof(struct metric *)); - new_instance->max_n_metric_master = instance->max_n_metric_master; - new_instance->n_metric_master = instance->n_metric_master; - for (size_t i = 0; i < instance->n_metric_master; i++) { - new_instance->metric_masters[i] = metric_fork(instance->metric_masters[i]); - } - new_instance->metric_name_id_map = name_id_map_copy(instance->metric_name_id_map); + new_instance->cube_manager = cube_manager_fork(instance->cube_manager); return new_instance; } -void calibrate_metrics_in_instance(const struct fieldstat *master, struct fieldstat *replica) +void fieldstat_calibrate(const struct fieldstat *master, struct fieldstat *replica) { - if (replica->max_n_metric_master < master->max_n_metric_master) { - replica->metric_masters = (struct metric **)realloc(replica->metric_masters, sizeof(struct metric *) * master->max_n_metric_master); - memset(replica->metric_masters + replica->max_n_metric_master, 0, sizeof(struct metric *) * (master->max_n_metric_master - replica->max_n_metric_master)); - replica->max_n_metric_master = master->max_n_metric_master; - } - - size_t longer_arr_len = master->n_metric_master > replica->n_metric_master ? master->n_metric_master : replica->n_metric_master; - for (size_t i = 0; i < longer_arr_len; i++) { - const struct metric *metric_master = i >= master->n_metric_master ? NULL : master->metric_masters[i]; - struct metric *metric_target = i >= replica->n_metric_master ? NULL : replica->metric_masters[i]; - if (metric_master == NULL && metric_target == NULL) { - continue; - } - if (metric_master == NULL && metric_target != NULL) { - metric_free(metric_target); - replica->metric_masters[i] = NULL; - continue; - } - if (metric_master != NULL && metric_target == NULL) { - const struct metric *metric_dup = metric_fork(metric_master); - add_metric_to_instance(replica, metric_dup, i); - continue; - } - if (metric_get_type(metric_master) != metric_get_type(metric_target) || - strcmp(metric_get_name(metric_master), metric_get_name(metric_target)) != 0 - ) { - metric_free(metric_target); - const struct metric *metric_dup = metric_fork(metric_master); - add_metric_to_instance(replica, metric_dup, i); - continue; - } - - // metric same, no need to do anything - } - - replica->n_metric_master = master->n_metric_master; -} - -int fieldstat_calibrate(const struct fieldstat *master, struct fieldstat *replica) -{ - if (master == NULL || replica == NULL) { - return FS_ERR_NULL_HANDLER; - } - - if (replica->max_n_cube < master->max_n_cube) { - replica->cube = (struct fs_cube **)realloc(replica->cube, sizeof(struct fs_cube *) * master->max_n_cube); - memset(replica->cube + replica->max_n_cube, 0, sizeof(struct fs_cube *) * (master->max_n_cube - replica->max_n_cube)); - replica->cube_version = (unsigned long *)realloc(replica->cube_version, sizeof(unsigned long) * master->max_n_cube); - memset(replica->cube_version + replica->max_n_cube, 0, sizeof(unsigned long) * (master->max_n_cube - replica->max_n_cube)); - replica->max_n_cube = master->max_n_cube; - } - - size_t len_master = master->valid_cube_arr_length; - size_t len_replica = replica->valid_cube_arr_length; - size_t longer_arr_len = len_master > len_replica ? len_master : len_replica; - for (size_t i = 0; i < longer_arr_len; i++) { - const struct fs_cube *cube_master = i >= len_master ? NULL : master->cube[i]; - const struct fs_cube *cube_target = i >= len_replica ? NULL : replica->cube[i]; - - if (cube_master == NULL && cube_target == NULL) { - continue; - } - if (cube_master == NULL && cube_target != NULL) { - fieldstat_cube_free_contents(replica, i); - continue; - } - if (cube_master != NULL && cube_target == NULL) { - struct fs_cube *cube_dup = fieldstat_cube_fork(cube_master); - add_cube_to_position(replica, cube_dup, i); - continue; - } - - if (master->cube_version[i] == replica->cube_version[i] && tag_hash_key_cmp(cube_master->key_tag, cube_target->key_tag) == 0) { - continue; - } - - fieldstat_cube_free_contents(replica, i); - struct fs_cube *cube_dup = fieldstat_cube_fork(cube_master); - add_cube_to_position(replica, cube_dup, i); - } - - memcpy(replica->cube_version, master->cube_version, sizeof(unsigned long) * master->max_n_cube); - replica->valid_cube_arr_length = master->valid_cube_arr_length; - - cube_manager_calibrate(replica->shared_tag_cube_manager, master->shared_tag_cube_manager); - calibrate_metrics_in_instance(master, replica); - - return FS_OK; + cube_manager_calibrate(replica->cube_manager, master->cube_manager); } /* -------------------------------------------------------------------------- */ @@ -1042,374 +225,128 @@ int fieldstat_calibrate(const struct fieldstat *master, struct fieldstat *replic /* -------------------------------------------------------------------------- */ void fieldstat_get_cubes(const struct fieldstat *instance, int **cube_ids, int *n_cube) { - if (instance == NULL || instance->valid_cube_arr_length == 0) { - *cube_ids = NULL; - *n_cube = 0; - return; - } - int all_available_cube_count = 0; - - int *tmp_ids = (int *)malloc(sizeof(int) * instance->valid_cube_arr_length); - for (int i = 0; i < instance->valid_cube_arr_length; i++) { - const struct fs_cube *tmp_cube = instance->cube[i]; - if (tmp_cube == NULL) { - continue; - } - tmp_ids[all_available_cube_count] = i; - all_available_cube_count ++; - } - if (all_available_cube_count == 0) { - free(tmp_ids); - *cube_ids = NULL; - *n_cube = 0; - return; - } - - *cube_ids = (int *)malloc(sizeof(int) * all_available_cube_count); - memcpy(*cube_ids, tmp_ids, sizeof(int) * all_available_cube_count); - *n_cube = all_available_cube_count; - - free(tmp_ids); + cube_manager_list(instance->cube_manager, cube_ids, n_cube); } -int fieldstat_get_metrics_used_by_cube(const struct fieldstat *instance, int cube_id, int **metric_id_out, size_t *n_metric) +struct field_list *fieldstat_cube_get_dimensions(const struct fieldstat *instance, int cube_id) { - *metric_id_out = NULL; - *n_metric = 0; - if (instance == NULL) { - return FS_ERR_NULL_HANDLER; - } - if (cube_id >= instance->valid_cube_arr_length || cube_id < 0) { - return FS_ERR_INVALID_CUBE_ID; - } - const struct fs_cube *cube = instance->cube[cube_id]; + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); if (cube == NULL) { - return FS_ERR_INVALID_CUBE_ID; - } - - int all_available_metric_count = 0; - if (cube->valid_metric_arr_len == 0) { - return FS_OK; - } - int *tmp_ids = (int *)malloc(sizeof(int) * cube->valid_metric_arr_len); - for (int i = 0; i < cube->valid_metric_arr_len; i++) { - const struct metric *tmp_metric = cube->metrics[i]; - if (tmp_metric == NULL) { - continue; - } - tmp_ids[all_available_metric_count] = i; - all_available_metric_count ++; - } - if (all_available_metric_count == 0) { - free(tmp_ids); - return FS_OK; - } - - *metric_id_out = tmp_ids; - *n_metric = all_available_metric_count; - - return FS_OK; -} - -void fieldstat_get_metrics(const struct fieldstat *instance, int **metric_id_out, size_t *n_metric) -{ - if (instance == NULL || instance->n_metric_master == 0) { - *metric_id_out = NULL; - *n_metric = 0; - return; - } - - int *tmp_ids = (int *)malloc(sizeof(int) * instance->n_metric_master); - *metric_id_out = tmp_ids; - *n_metric = instance->n_metric_master; - for (int i = 0; i < instance->n_metric_master; i++) { - tmp_ids[i] = i; + return NULL; } - - return; + + return cube_get_identifier(cube); } -void fieldstat_get_cells_used_by_metric(const struct fieldstat *instance, int cube_id, int metric_id, - struct fieldstat_tag_list **tag_list, size_t *n_cell) +int fieldstat_counter_get(const struct fieldstat *instance, int cube_id, const struct field_list *cell_dimensions, int metric_id, long long *value) { - *tag_list = NULL; - *n_cell = 0; - int ret = FS_OK; - const struct metric *metric = fieldstat_find_metric(instance, cube_id, metric_id, &ret); - if (metric == NULL) { - return; - } - - size_t n_cell_ret = 0; - int *cell_ids = NULL; - metric_get_cell_ids(metric, &cell_ids, &n_cell_ret); - if (n_cell_ret == 0) { - return; - } - *n_cell = n_cell_ret; - - struct fieldstat_tag_list *tag_list_ret = (struct fieldstat_tag_list *)malloc(sizeof(struct fieldstat_tag_list) * n_cell_ret); - *tag_list = tag_list_ret; - for (int i = 0; i < n_cell_ret; i++) { - const struct tag_hash_key *tag_key = cell_manager_get_tag_by_cell_id(instance->cube[cube_id]->cell_manager, cell_ids[i]); - if (tag_key == NULL) { - continue; - } - tag_hash_key_convert_to_fieldstat_tag(tag_key, &(tag_list_ret[i].tag), &(tag_list_ret[i].n_tag)); + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + if (cube == NULL) { + return FS_ERR_INVALID_CUBE_ID; } - - free(cell_ids); + + return cube_counter_get(cube, metric_id, cell_dimensions, value); } -struct fieldstat_tag_list *fieldstat_get_shared_tags(const struct fieldstat *instance, int cube_id) +int fieldstat_hll_get(const struct fieldstat *instance, int cube_id, const struct field_list *cell_dimensions, int metric_id, double *value) { - if (instance == NULL || cube_id >= instance->valid_cube_arr_length || cube_id < 0) { - return NULL; - } - struct fs_cube *cube = instance->cube[cube_id]; + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); if (cube == NULL) { - return NULL; - } - - struct fieldstat_tag_list *tag_list = (struct fieldstat_tag_list *)malloc(sizeof(struct fieldstat_tag_list)); - - if (cube->n_shared_tags == 0) { - tag_list->tag = NULL; - tag_list->n_tag = 0; - return tag_list; - } - - tag_list->tag = (struct fieldstat_tag *)malloc(sizeof(struct fieldstat_tag) * cube->n_shared_tags); - for (int i = 0; i < cube->n_shared_tags; i++) { - struct fieldstat_tag *tag_dest = &(tag_list->tag[i]); - struct fieldstat_tag *tag_src = &(cube->shared_tags[i]); - tag_dest->key = strdup(tag_src->key); - tag_dest->type = tag_src->type; - switch (tag_src->type) { - case TAG_INTEGER: - tag_dest->value_longlong = tag_src->value_longlong; - break; - case TAG_CSTRING: - tag_dest->value_str = strdup(tag_src->value_str); - break; - case TAG_DOUBLE: - tag_dest->value_double = tag_src->value_double; - break; - default: - break; - } + return FS_ERR_INVALID_CUBE_ID; } - tag_list->n_tag = cube->n_shared_tags; - - return tag_list; -} + + int ret = cube_hll_get(cube, metric_id, cell_dimensions, value); -int get_cell_id_by_tag_list(const struct fieldstat *instance, int cube_id, const struct fieldstat_tag_list *tags) -{ - struct tag_hash_key tag_key; - tag_hash_key_init_with_fieldstat_tag(&tag_key, tags->tag, tags->n_tag, false); - int cell_id = cell_manager_find(instance->cube[cube_id]->cell_manager, &tag_key); - return cell_id; + return ret; } -int fieldstat_counter_get(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tags, long long *value) +long long fieldstat_histogram_value_at_percentile(const struct fieldstat *instance, int cube_id, const struct field_list *cell_dimensions, int metric_id, double percentile) { - int ret; - *value = 0; - const struct metric *metric = fieldstat_find_metric(instance, cube_id, metric_id, &ret); - if (ret != FS_OK) { - return ret; - } - - if (metric == NULL || metric_get_type(metric) != METRIC_TYPE_COUNTER) { - return FS_ERR_INVALID_METRIC_ID; - } - - int cell_id = get_cell_id_by_tag_list(instance, cube_id, tags); - if (cell_id == -1) { - return FS_ERR_INVALID_TAG; + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + if (cube == NULL) { + return FS_ERR_INVALID_CUBE_ID; } - - ret = metric_counter_get(metric, cell_id, value); + long long value; + int ret = cube_histogram_value_at_percentile(cube, metric_id, cell_dimensions, percentile, &value); if (ret < 0) { - return FS_ERR_INVALID_TAG; - } - - return FS_OK; -} - -int fieldstat_hll_get(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tags, double *value) -{ - int ret; - const struct metric *metric = fieldstat_find_metric(instance, cube_id, metric_id, &ret); - if (ret != FS_OK) { return ret; } - if (metric == NULL || metric_get_type(metric) != METRIC_TYPE_HLL) { - return FS_ERR_INVALID_METRIC_ID; - } - - int cell_id = get_cell_id_by_tag_list(instance, cube_id, tags); - if (cell_id == -1) { - return FS_ERR_INVALID_TAG; - } - - double ret2 = metric_hll_get(metric, cell_id); - if (ret2 < 0) { - return FS_ERR_INVALID_TAG; - } - *value = ret2; - return FS_OK; + return value; } -long long fieldstat_hist_value_at_percentile(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tags, double percentile) +long long fieldstat_histogram_count_le_value(const struct fieldstat *instance, int cube_id, const struct field_list *cell_dimensions, int metric_id, long long value) { - int ret; - const struct metric *metric = fieldstat_find_metric(instance, cube_id, metric_id, &ret); - if (ret != FS_OK) { - return ret; - } - if (metric == NULL || metric_get_type(metric) != METRIC_TYPE_HISTOGRAM) { - return FS_ERR_INVALID_METRIC_ID; - } - - int cell_id = get_cell_id_by_tag_list(instance, cube_id, tags); - if (cell_id == -1) { - return FS_ERR_INVALID_TAG; - } - long long ret2 = metric_histogram_value_at_percentile(metric, cell_id, percentile); - if (ret2 < 0) { - return FS_ERR_INVALID_TAG; + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + if (cube == NULL) { + return FS_ERR_INVALID_CUBE_ID; } - - return ret2; -} - -long long fieldstat_hist_count_le_value(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tags, long long value) -{ - int ret; - const struct metric *metric = fieldstat_find_metric(instance, cube_id, metric_id, &ret); - if (ret != FS_OK) { + long long count; + int ret = cube_histogram_count_le_value(cube, metric_id, cell_dimensions, value, &count); + if (ret < 0) { return ret; } - if (metric == NULL || metric_get_type(metric) != METRIC_TYPE_HISTOGRAM) { - return FS_ERR_INVALID_METRIC_ID; - } - - int cell_id = get_cell_id_by_tag_list(instance, cube_id, tags); - if (cell_id == -1) { - return FS_ERR_INVALID_TAG; - } - - long long ret2 = metric_histogram_count_le_value(metric, cell_id, value); - if (ret2 < 0) { - return FS_ERR_INVALID_TAG; - } - - return ret2; + return count; } -void fieldstat_get_serialized_blob(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tags, char **blob, size_t *blob_size) +void fieldstat_metric_get_serialization_as_base64(const struct fieldstat *instance, int cube_id, int metric_id, const struct field_list *cell_dimensions, char **blob, size_t *blob_size) { - *blob = NULL; - *blob_size = 0; - int ret; - const struct metric *metric = fieldstat_find_metric(instance, cube_id, metric_id, &ret); - if (ret != FS_OK) { - return; - } - if (metric == NULL) { - return; - } - enum metric_type type = metric_get_type(metric); - if (!(type == METRIC_TYPE_HLL || type == METRIC_TYPE_HISTOGRAM)) { - return; - } - - int cell_id = get_cell_id_by_tag_list(instance, cube_id, tags); - if (cell_id == -1) { + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + if (cube == NULL) { + *blob = NULL; + *blob_size = 0; return; } - metric_get_plain_blob(metric, cell_id, blob, blob_size); + cube_get_serialization_as_base64(cube, metric_id, cell_dimensions, blob, blob_size); } -void fieldstat_tag_list_arr_free(struct fieldstat_tag_list *tag_list, size_t n_cell) +void fieldstat_field_list_arr_free(struct field_list *field_lists, size_t n_field_list) { - if (tag_list == NULL) { + if (field_lists == NULL) { return; } - for (int i = 0; i < n_cell; i++) { - fieldstat_free_tag_array(tag_list[i].tag, tag_list[i].n_tag); + for (int i = 0; i < n_field_list; i++) { + for (size_t j = 0; j < field_lists[i].n_field; j++) { + struct field *field = &(field_lists[i].field[j]); + free((char *)field->key); + if (field->type == FIELD_VALUE_CSTRING) { + free((char *)field->value_str); + } + } + free(field_lists[i].field); } - free(tag_list); + free(field_lists); } -const char *fieldstat_get_metric_name(const struct fieldstat *instance, int metric_id) +//uninitialized + +const char *fieldstat_get_metric_name(const struct fieldstat *instance, int cube_id, int metric_id) { - if (metric_id < 0 || metric_id >= instance->n_metric_master) { - return NULL; - } - const struct metric *metric = instance->metric_masters[metric_id]; - if (metric == NULL) { + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + if (cube == NULL) { return NULL; } - - return metric_get_name(metric); + return cube_get_metric_name(cube, metric_id); } -enum metric_type fieldstat_get_metric_type(const struct fieldstat *instance, int metric_id) +enum metric_type fieldstat_get_metric_type(const struct fieldstat *instance, int cube_id, int metric_id) { - if (instance == NULL || metric_id < 0 || metric_id >= instance->n_metric_master) { - return (enum metric_type)(-1); - } - const struct metric *metric = instance->metric_masters[metric_id]; - if (metric == NULL) { + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + if (cube == NULL) { return (enum metric_type)(-1); } - - return metric_get_type(metric); + return cube_get_metric_type(cube, metric_id); } -void fieldstat_get_cells_used_by_cube(const struct fieldstat *instance, int cube_id, struct fieldstat_tag_list **tag_list, size_t *n_cell) +void fieldstat_cube_get_cells(const struct fieldstat *instance, int cube_id, struct field_list **cell_dimensions, size_t *n_cell) { - if (instance == NULL || cube_id < 0 || cube_id >= instance->valid_cube_arr_length) { - return; - } - const struct fs_cube *cube = instance->cube[cube_id]; + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); if (cube == NULL) { return; } - int arr_len = 0; - const struct tag_hash_key **tags_discontinuous = cell_manager_dump(cube->cell_manager, &arr_len); - if (arr_len == 0) { - *tag_list = NULL; - *n_cell = 0; - return; - } - - struct fieldstat_tag_list *tag_list_ret = (struct fieldstat_tag_list *)malloc(sizeof(struct fieldstat_tag_list) * arr_len); - int n_cell_ret = 0; - - for (int i = 0; i < arr_len; i++) { - const struct tag_hash_key *tag_key = tags_discontinuous[i]; - if (tag_key == NULL) { - continue; - } - tag_hash_key_convert_to_fieldstat_tag(tag_key, &(tag_list_ret[n_cell_ret].tag), &(tag_list_ret[n_cell_ret].n_tag)); - n_cell_ret++; - } - if (n_cell_ret == 0) { - free(tag_list_ret); - *tag_list = NULL; - *n_cell = 0; - return; - } - - *tag_list = tag_list_ret; - *n_cell = n_cell_ret; + cube_get_cells(cube, cell_dimensions, n_cell); } int fieldstat_get_used_sampling(const struct fieldstat *instance, int cube_id) @@ -1417,52 +354,36 @@ int fieldstat_get_used_sampling(const struct fieldstat *instance, int cube_id) if (instance == NULL) { return FS_ERR_NULL_HANDLER; } - if (cube_id < 0 || cube_id >= instance->valid_cube_arr_length) { - return FS_ERR_INVALID_CUBE_ID; - } - const struct fs_cube *cube = instance->cube[cube_id]; + + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); if (cube == NULL) { return FS_ERR_INVALID_CUBE_ID; } - return cell_manager_get_cardinality(cube->cell_manager); + return cube_get_cell_count(cube); } -int fieldstat_find_cube(const struct fieldstat *instance, const struct fieldstat_tag *shared_tags, size_t n_shared_tags) +int fieldstat_find_cube(const struct fieldstat *instance, const struct field *cube_dimensions, size_t n_dimensions) { if (instance == NULL) { return FS_ERR_NULL_HANDLER; } - const struct cube_manager *tag_cube_id_map = instance->shared_tag_cube_manager; - struct tag_hash_key shared_tag_key; - tag_hash_key_init_with_fieldstat_tag(&shared_tag_key, shared_tags, n_shared_tags, false); - int cube_id = cube_manager_find(tag_cube_id_map, &shared_tag_key); + + int cube_id = cube_manager_find(instance->cube_manager, cube_dimensions, n_dimensions); if (cube_id == -1) { - return FS_ERR_INVALID_KEY; + return FS_ERR_INVALID_DIMENSION; } return cube_id; } -int fieldstat_get_cube_mode(const struct fieldstat *instance, int cube_id, enum sampling_mode *mode, int *primary_metric_id) +void fieldstat_get_metric_in_cell(const struct fieldstat *instance, int cube_id, const struct field_list *cell_dimensions, int **metric_id_out, size_t *n_metric_out) { - if (instance == NULL) { - return FS_ERR_NULL_HANDLER; - } - if (cube_id < 0 || cube_id >= instance->valid_cube_arr_length) { - return FS_ERR_INVALID_CUBE_ID; - } - const struct fs_cube *cube = instance->cube[cube_id]; - if (cube == NULL) { - return FS_ERR_INVALID_CUBE_ID; - } - - *mode = cube->sampling_mode; - if (cube->sampling_mode == SAMPLING_MODE_TOPK) { - *primary_metric_id = cube->primary_metric_id; - } else { - *primary_metric_id = -1; - } + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + return cube_get_metrics_in_cell(cube, cell_dimensions, metric_id_out, n_metric_out); +} - return FS_OK; -}
\ No newline at end of file +void fieldstat_cube_get_metrics(const struct fieldstat *instance, int cube_id, int **metric_id_out, size_t *n_metric) { + const struct cube *cube = cube_manager_get_cube_by_id(instance->cube_manager, cube_id); + cube_get_metrics(cube, metric_id_out, n_metric); +} diff --git a/src/fieldstat_easy.c b/src/fieldstat_easy.c index 9319a41..f32dbd9 100644 --- a/src/fieldstat_easy.c +++ b/src/fieldstat_easy.c @@ -114,7 +114,7 @@ void *fs_easy_output_thread(void *arg) // return void * for pthread_create check return NULL; // return void * for pthread_create check only } -struct fieldstat_easy *fieldstat_easy_new(int max_thread_num, const char *name, const struct fieldstat_tag *tags, size_t n_tag) { +struct fieldstat_easy *fieldstat_easy_new(int max_thread_num, const char *name, const struct field *global_dimensions, size_t n_dimension) { if (max_thread_num <= 0) { return NULL; } @@ -122,12 +122,13 @@ struct fieldstat_easy *fieldstat_easy_new(int max_thread_num, const char *name, fse->fsu = malloc(sizeof(struct fs_easy_thread) * max_thread_num); fse->max_thread_num = max_thread_num; fse->delta = fieldstat_new(); - fieldstat_create_cube(fse->delta, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); + fieldstat_cube_create(fse->delta, NULL, 0); + fieldstat_cube_set_sampling(fse->delta, 0, SAMPLING_MODE_COMPREHENSIVE, 0, 0); fse->accumulate = fieldstat_fork(fse->delta); fse->exporter = fieldstat_json_exporter_new(); - if (tags != NULL && n_tag > 0) { - fieldstat_json_exporter_set_global_tag(fse->exporter, tags, n_tag); + if (global_dimensions != NULL && n_dimension > 0) { + fieldstat_json_exporter_set_global_tag(fse->exporter, global_dimensions, n_dimension); } if (name != NULL) { fieldstat_json_exporter_set_name(fse->exporter, name); @@ -192,17 +193,17 @@ int fieldstat_easy_register_counter(struct fieldstat_easy *fse, const char *name pthread_spin_lock(&fse->fsu[i].lock); } - int ret = fieldstat_register_counter(fse->fsu[0].active, name); // try to register + int ret = fieldstat_register_counter(fse->fsu[0].active, 0, name); // try to register if (ret < 0) { for (int i = 0; i < fse->max_thread_num; i++) { pthread_spin_unlock(&fse->fsu[i].lock); } return ret; } - fieldstat_register_counter(fse->fsu[0].read_only, name); + fieldstat_register_counter(fse->fsu[0].read_only, 0, name); for (int i = 1; i < fse->max_thread_num; i++) { - fieldstat_register_counter(fse->fsu[i].active, name); - fieldstat_register_counter(fse->fsu[i].read_only, name); + fieldstat_register_counter(fse->fsu[i].active, 0, name); + fieldstat_register_counter(fse->fsu[i].read_only, 0, name); } for (int i = 0; i < fse->max_thread_num; i++) { @@ -218,17 +219,17 @@ int fieldstat_easy_register_histogram(struct fieldstat_easy *fse, const char *na pthread_spin_lock(&fse->fsu[i].lock); } - int ret = fieldstat_register_hist(fse->fsu[0].active, name, lowest_trackable_value, highest_trackable_value, significant_figures); // try to register + int ret = fieldstat_register_histogram(fse->fsu[0].active, 0, name, lowest_trackable_value, highest_trackable_value, significant_figures); // try to register if (ret < 0) { for (int i = 0; i < fse->max_thread_num; i++) { pthread_spin_unlock(&fse->fsu[i].lock); } return ret; } - fieldstat_register_hist(fse->fsu[0].read_only, name, lowest_trackable_value, highest_trackable_value, significant_figures); + fieldstat_register_histogram(fse->fsu[0].read_only, 0, name, lowest_trackable_value, highest_trackable_value, significant_figures); for (int i = 1; i < fse->max_thread_num; i++) { - fieldstat_register_hist(fse->fsu[i].active, name, lowest_trackable_value, highest_trackable_value, significant_figures); - fieldstat_register_hist(fse->fsu[i].read_only, name, lowest_trackable_value, highest_trackable_value, significant_figures); + fieldstat_register_histogram(fse->fsu[i].active, 0, name, lowest_trackable_value, highest_trackable_value, significant_figures); + fieldstat_register_histogram(fse->fsu[i].read_only, 0, name, lowest_trackable_value, highest_trackable_value, significant_figures); } for (int i = 0; i < fse->max_thread_num; i++) { @@ -250,35 +251,35 @@ struct timeval get_current_timestamp() struct fieldstat *merge_all_instance(struct fieldstat_easy *fse) { - struct fieldstat *dst = fieldstat_new(); + struct fieldstat *dest = fieldstat_new(); // collect all the data recorded since last passive output, if passive output happened. Otherwise, its the data since the program started. for (int i = 0; i < fse->max_thread_num; i++) { pthread_spin_lock(&fse->fsu[i].lock); - fieldstat_merge(dst, fse->fsu[i].active); + fieldstat_merge(dest, fse->fsu[i].active); pthread_spin_unlock(&fse->fsu[i].lock); } // add the outputted data if (fse->output_thread_running) { pthread_spin_lock(&fse->outputting_lock); - fieldstat_merge(dst, fse->accumulate); + fieldstat_merge(dest, fse->accumulate); pthread_spin_unlock(&fse->outputting_lock); } - return dst; + return dest; } // output an json string for accumulated data void fieldstat_easy_output(struct fieldstat_easy *fse, char **buff, size_t *buff_len) { - struct fieldstat *dst = merge_all_instance(fse); + struct fieldstat *dest = merge_all_instance(fse); struct timeval timestamp = get_current_timestamp(); - *buff = fieldstat_json_exporter_export(fse->exporter, dst, ×tamp); + *buff = fieldstat_json_exporter_export_flat(fse->exporter, dest, ×tamp); if (*buff == NULL) { *buff = strdup("[]"); } - fieldstat_free(dst); + fieldstat_free(dest); *buff_len = strlen(*buff); } @@ -286,10 +287,10 @@ void fieldstat_easy_output(struct fieldstat_easy *fse, char **buff, size_t *buff void fieldstat_easy_output_array(struct fieldstat_easy *fse, char ***json_objects, size_t *n_object) { struct timeval timestamp = get_current_timestamp(); - struct fieldstat *dst = merge_all_instance(fse); + struct fieldstat *dest = merge_all_instance(fse); - fieldstat_json_exporter_export_array(fse->exporter, dst, ×tamp, json_objects, n_object); - fieldstat_free(dst); + fieldstat_json_exporter_export_flat_array(fse->exporter, dest, ×tamp, json_objects, n_object); + fieldstat_free(dest); } int fieldstat_easy_output_array_and_reset(struct fieldstat_easy *fse, char ***json_objects, size_t *n_object) @@ -299,20 +300,20 @@ int fieldstat_easy_output_array_and_reset(struct fieldstat_easy *fse, char ***js return -1; } - struct fieldstat *dst = fieldstat_new(); + struct fieldstat *dest = fieldstat_new(); for (int i = 0; i < fse->max_thread_num; i++) { rcu_reclaim_handler(fse->fsu + i); - fieldstat_merge(dst, fse->fsu[i].read_only); + fieldstat_merge(dest, fse->fsu[i].read_only); } struct timeval timestamp = get_current_timestamp(); - fieldstat_json_exporter_export_array(fse->exporter, dst, ×tamp, json_objects, n_object); - fieldstat_free(dst); + fieldstat_json_exporter_export_flat_array(fse->exporter, dest, ×tamp, json_objects, n_object); + fieldstat_free(dest); return 0; } -int fieldstat_easy_counter_incrby(struct fieldstat_easy *fse, int thread_id, int metric_id, const struct fieldstat_tag *tags, size_t n_tag, long long increment) +int fieldstat_easy_counter_incrby(struct fieldstat_easy *fse, int thread_id, int metric_id, const struct field *dimensions, size_t n_dimensions, long long increment) { if (thread_id < 0) { return -1; @@ -322,13 +323,13 @@ int fieldstat_easy_counter_incrby(struct fieldstat_easy *fse, int thread_id, int } pthread_spin_lock(&fse->fsu[thread_id].lock); - int ret = fieldstat_counter_incrby(fse->fsu[thread_id].active, 0, metric_id, tags, n_tag, increment); + int ret = fieldstat_counter_incrby(fse->fsu[thread_id].active, 0, metric_id, dimensions, n_dimensions, increment); pthread_spin_unlock(&fse->fsu[thread_id].lock); return ret; } -int fieldstat_easy_counter_set(struct fieldstat_easy *fse, int thread_id, int metric_id, const struct fieldstat_tag *tags, size_t n_tag, long long value) +int fieldstat_easy_counter_set(struct fieldstat_easy *fse, int thread_id, int metric_id, const struct field *dimensions, size_t n_dimensions, long long value) { if (thread_id < 0) { return -1; @@ -338,13 +339,13 @@ int fieldstat_easy_counter_set(struct fieldstat_easy *fse, int thread_id, int me } pthread_spin_lock(&fse->fsu[thread_id].lock); - int ret = fieldstat_counter_set(fse->fsu[thread_id].active, 0, metric_id, tags, n_tag, value); + int ret = fieldstat_counter_set(fse->fsu[thread_id].active, 0, metric_id, dimensions, n_dimensions, value); pthread_spin_unlock(&fse->fsu[thread_id].lock); return ret; } -int fieldstat_easy_histogram_record(struct fieldstat_easy *fse, int thread_id, int metric_id, const struct fieldstat_tag *tags, size_t n_tag, long long value) +int fieldstat_easy_histogram_record(struct fieldstat_easy *fse, int thread_id, int metric_id, const struct field *dimensions, size_t n_dimensions, long long value) { if (thread_id < 0) { return -1; @@ -354,13 +355,8 @@ int fieldstat_easy_histogram_record(struct fieldstat_easy *fse, int thread_id, i } pthread_spin_lock(&fse->fsu[thread_id].lock); - int ret = fieldstat_hist_record(fse->fsu[thread_id].active, 0, metric_id, tags, n_tag, value); + int ret = fieldstat_histogram_record(fse->fsu[thread_id].active, 0, metric_id, dimensions, n_dimensions, value); pthread_spin_unlock(&fse->fsu[thread_id].lock); return ret; } - -void fieldstat_easy_enable_delta_in_active_output(struct fieldstat_easy *fse) -{ - fieldstat_json_exporter_enable_delta(fse->exporter); -}
\ No newline at end of file diff --git a/src/metrics/st_hyperloglog.c b/src/metrics/hyperloglog.c index 5085671..ea05887 100644 --- a/src/metrics/st_hyperloglog.c +++ b/src/metrics/hyperloglog.c @@ -3,7 +3,7 @@ * For our needs, we always use a dense representation and avoid the sparse/dense conversions. * */ -#include "st_hyperloglog.h" +#include "hyperloglog.h" #include "xxhash/xxhash.h" #include "crdt_utils.h" @@ -14,16 +14,18 @@ #include <stdlib.h> #include "base64/b64.h" -const size_t BLOB_HDR_SIZE= offsetof(struct ST_hyperloglog, registers); +#include "hll_common.h" +const size_t BLOB_HDR_SIZE= offsetof(struct hyperloglog, registers); -struct ST_hyperloglog *ST_hyperloglog_new(unsigned char precision) + +struct hyperloglog *hyperloglog_new(unsigned char precision) { // Ensure the precision is somewhat sane if (precision < HLL_MIN_PRECISION || precision > HLL_MAX_PRECISION) return NULL; - struct ST_hyperloglog *h=ALLOC(struct ST_hyperloglog, 1); + struct hyperloglog *h=ALLOC(struct hyperloglog, 1); h->cfg.precision = precision; @@ -37,24 +39,8 @@ struct ST_hyperloglog *ST_hyperloglog_new(unsigned char precision) h->registers = ALLOC(uint32_t, words); return h; } -void ST_hyperloglog_configure(struct ST_hyperloglog *h, unsigned char precision, int time_window_seconds, const struct timeval now) -{ - if(h->cfg.precision != precision) - { - free(h->registers); - // Determine how many registers are needed - int reg = NUM_REG(precision); - - // Get the full words required - int words = INT_CEIL(reg, REG_PER_WORD); - // Allocate and zero out the registers - h->registers = ALLOC(uint32_t, words); - h->cfg.precision=precision; - } - return; -} -void ST_hyperloglog_free(struct ST_hyperloglog *h) +void hyperloglog_free(struct hyperloglog *h) { free(h->registers); h->registers=NULL; @@ -62,14 +48,14 @@ void ST_hyperloglog_free(struct ST_hyperloglog *h) return; } -static int get_register(const struct ST_hyperloglog *h, int idx) { +static int get_register(const struct hyperloglog *h, int idx) { uint32_t word = *(h->registers + (idx / REG_PER_WORD)); word = word >> REG_WIDTH * (idx % REG_PER_WORD); return word & ((1 << REG_WIDTH) - 1); } -static void set_register(const struct ST_hyperloglog *h, int idx, int val) { +static void set_register(const struct hyperloglog *h, int idx, int val) { uint32_t *word = h->registers + (idx / REG_PER_WORD); // Shift the val into place @@ -82,7 +68,7 @@ static void set_register(const struct ST_hyperloglog *h, int idx, int val) { return; } -static void reset_register(const struct ST_hyperloglog *h, int idx) +static void reset_register(const struct hyperloglog *h, int idx) { uint32_t *word = h->registers + (idx / REG_PER_WORD); unsigned shift = REG_WIDTH * (idx % REG_PER_WORD); @@ -90,7 +76,7 @@ static void reset_register(const struct ST_hyperloglog *h, int idx) *word &= ~val_mask; } -void ST_hyperloglog_reset(struct ST_hyperloglog *h) +void hyperloglog_reset(struct hyperloglog *h) { int n_register=NUM_REG(h->cfg.precision); @@ -100,7 +86,7 @@ void ST_hyperloglog_reset(struct ST_hyperloglog *h) return; } -int hll_add_hash(struct ST_hyperloglog *h, uint64_t hash) +int hyperloglog_add_hash(struct hyperloglog *h, uint64_t hash) { // Determine the index using the first p bits int idx = hash >> (64 - h->cfg.precision); @@ -119,30 +105,30 @@ int hll_add_hash(struct ST_hyperloglog *h, uint64_t hash) return 0; } -int ST_hyperloglog_add(struct ST_hyperloglog *h, const char *key, size_t keylen) +int hyperloglog_add(struct hyperloglog *h, const char *key, size_t keylen) { uint64_t hash=0; hash=XXH3_64bits_withSeed(key, keylen, 171); // Add the hashed value - return hll_add_hash(h, hash); + return hyperloglog_add_hash(h, hash); } // https://djhworld.github.io/hyperloglog/merging/ -int ST_hyperloglog_merge(struct ST_hyperloglog *dst, const struct ST_hyperloglog *src) +int hyperloglog_merge(struct hyperloglog *dest, const struct hyperloglog *src) { - if(dst->cfg.precision != src->cfg.precision) return -1; - int n_register=NUM_REG(dst->cfg.precision); + if(dest->cfg.precision != src->cfg.precision) return -1; + int n_register=NUM_REG(dest->cfg.precision); int s_reg=0, d_reg=0; for(int i=0; i<n_register; i++) { s_reg=get_register(src, i); - d_reg=get_register(dst, i); - set_register(dst, i, MAX(s_reg, d_reg)); + d_reg=get_register(dest, i); + set_register(dest, i, MAX(s_reg, d_reg)); } return 0; } -size_t ST_hyperloglog_serialized_size(const struct ST_hyperloglog *h) +size_t hyperloglog_serialized_size(const struct hyperloglog *h) { size_t sz=0; size_t num_reg = NUM_REG(h->cfg.precision); @@ -153,13 +139,13 @@ size_t ST_hyperloglog_serialized_size(const struct ST_hyperloglog *h) return sz; } -void ST_hyperloglog_serialize(const struct ST_hyperloglog *h, char **blob, size_t *blob_sz) +void hyperloglog_serialize(const struct hyperloglog *h, char **blob, size_t *blob_sz) { size_t sz=0, offset=0; size_t num_reg = NUM_REG(h->cfg.precision); size_t words = INT_CEIL(num_reg, REG_PER_WORD); - sz = ST_hyperloglog_serialized_size(h); + sz = hyperloglog_serialized_size(h); char *buffer = ALLOC(char, sz); memcpy(buffer+offset, h, BLOB_HDR_SIZE); @@ -172,9 +158,9 @@ void ST_hyperloglog_serialize(const struct ST_hyperloglog *h, char **blob, size_ return; } -struct ST_hyperloglog *ST_hyperloglog_deserialize(const char *blob, size_t blob_sz) +struct hyperloglog *hyperloglog_deserialize(const char *blob, size_t blob_sz) { - struct ST_hyperloglog *h=ALLOC(struct ST_hyperloglog, 1); + struct hyperloglog *h=ALLOC(struct hyperloglog, 1); size_t offset=0; memcpy(h, blob, BLOB_HDR_SIZE); offset += BLOB_HDR_SIZE; @@ -187,43 +173,16 @@ struct ST_hyperloglog *ST_hyperloglog_deserialize(const char *blob, size_t blob_ return h; } -void ST_hyperloglog_serialize_for_networking(const struct ST_hyperloglog *h, char **blob, size_t *blob_sz) +void hyperloglog_serialize_into_base64(const struct hyperloglog *h, char **blob, size_t *blob_sz) { - size_t sz=0, offset=0; - size_t num_reg = NUM_REG(h->cfg.precision); - size_t words = INT_CEIL(num_reg, REG_PER_WORD); - - sz = ST_hyperloglog_serialized_size(h) + 1; // [precision][version][data... - unsigned char *buffer = ALLOC(unsigned char, sz); - - const unsigned char version = 1; - memcpy(buffer+offset, &version, 1); - offset += 1; - - unsigned char precision = h->cfg.precision; - memcpy(buffer+offset, &precision, 1); - offset += 1; - - for (int i = 0; i < words; i++) { - uint32_t word = h->registers[i]; - buffer[offset++] = word >> 24; - buffer[offset++] = word >> 16; - buffer[offset++] = word >> 8; - buffer[offset++] = word; - } - - char *enc = b64_encode(buffer, sz); - free(buffer); - *blob_sz = strlen(enc); - *blob = enc; - return; + hll_encode_into_base64(h->registers, h->cfg.precision, blob, blob_sz); } -void ST_hyperloglog_merge_blob(struct ST_hyperloglog *dst, const char *blob, size_t blob_sz) +void hyperloglog_merge_blob(struct hyperloglog *dest, const char *blob, size_t blob_sz) { - struct ST_hyperloglog *src=ST_hyperloglog_deserialize(blob, blob_sz); - ST_hyperloglog_merge(dst, src); - ST_hyperloglog_free(src); + struct hyperloglog *src=hyperloglog_deserialize(blob, blob_sz); + hyperloglog_merge(dest, src); + hyperloglog_free(src); return; } double g_switchThreshold[15] = {10, 20, 40, 80, 220, 400, 900, 1800, 3100, 6500, @@ -314,7 +273,7 @@ static double alpha(unsigned char precision) { /* * Computes the raw cardinality estimate */ -static double raw_estimate(const struct ST_hyperloglog *h, int *num_zero) +static double raw_estimate(const struct hyperloglog *h, int *num_zero) { unsigned char precision = h->cfg.precision; int num_reg = NUM_REG(precision); @@ -334,7 +293,7 @@ static double raw_estimate(const struct ST_hyperloglog *h, int *num_zero) * Estimates cardinality using a linear counting. * Used when some registers still have a zero value. */ -static double linear_count(const struct ST_hyperloglog *h, int num_zero) +static double linear_count(const struct hyperloglog *h, int num_zero) { int registers = NUM_REG(h->cfg.precision); return registers * @@ -364,7 +323,7 @@ static int binary_search(double val, int num, const double *array) { * empircal data collected by Google, from the * paper mentioned above. */ -static double bias_estimate(const struct ST_hyperloglog *h, double raw_est) { +static double bias_estimate(const struct hyperloglog *h, double raw_est) { // Determine the samples available int samples; int precision = h->cfg.precision; @@ -393,8 +352,8 @@ static double bias_estimate(const struct ST_hyperloglog *h, double raw_est) { else return (biases[idx] + biases[idx-1]) / 2; } - __attribute__ ((unused)) static double ST_HLL_count_no_sliding(const struct ST_hyperloglog *h) -{ + +double hyperloglog_count(const struct hyperloglog *h){ int num_zero = 0; int num_reg = NUM_REG(h->cfg.precision); double raw_est = raw_estimate(h, &num_zero); @@ -420,27 +379,6 @@ static double bias_estimate(const struct ST_hyperloglog *h, double raw_est) { return raw_est; } } -double ST_hyperloglog_count(const struct ST_hyperloglog *h) -{ - int num_zero=0; - double raw_est=raw_estimate(h, &num_zero); - raw_est*=alpha(h->cfg.precision); - - double num_reg=NUM_REG(h->cfg.precision); - if (raw_est <= 5 * num_reg) { - raw_est -= bias_estimate(h, raw_est); - } - if(raw_est<=5*num_reg/2) - { - if(num_zero) - return linear_count(h, num_zero); - } - if(raw_est>INT32_MAX/30) - { - return INT32_MIN*log(1-raw_est/INT32_MAX); - } - return raw_est; -} /** * Computes the upper bound on variance given @@ -449,7 +387,7 @@ double ST_hyperloglog_count(const struct ST_hyperloglog *h) * @return The expected variance in the count, * or zero on error. */ -double ST_hyperloglog_error_for_precision(unsigned char precision) +double hyperloglog_error_for_precision(unsigned char precision) { // Check that the error bound is sane if (precision < HLL_MIN_PRECISION || precision > HLL_MAX_PRECISION) diff --git a/src/metrics/st_hyperloglog.h b/src/metrics/hyperloglog.h index 98dd7a3..b1b2330 100644 --- a/src/metrics/st_hyperloglog.h +++ b/src/metrics/hyperloglog.h @@ -30,30 +30,30 @@ extern "C" #define REGISTER_SIZE(precision) INT_WIDTH*INT_CEIL(NUM_REG(precision), REG_PER_WORD) -struct ST_HLL_configuration +struct HLL_configuration { unsigned char precision; }; -struct ST_hyperloglog +struct hyperloglog { - struct ST_HLL_configuration cfg; + struct HLL_configuration cfg; uint32_t *registers; }; -struct ST_hyperloglog *ST_hyperloglog_new(unsigned char precision); -void ST_hyperloglog_free(struct ST_hyperloglog *h); +struct hyperloglog *hyperloglog_new(unsigned char precision); +void hyperloglog_free(struct hyperloglog *h); //Return 1 if at least 1 ST HyperLogLog internal register was altered. 0 otherwise. -int ST_hyperloglog_add(struct ST_hyperloglog *h, const char *key, size_t keylen); -void ST_hyperloglog_reset(struct ST_hyperloglog *h); -double ST_hyperloglog_count(const struct ST_hyperloglog *h); -size_t ST_hyperloglog_serialized_size(const struct ST_hyperloglog *h); -void ST_hyperloglog_serialize(const struct ST_hyperloglog *h, char **blob, size_t *blob_sz); -void ST_hyperloglog_serialize_for_networking(const struct ST_hyperloglog *h, char **blob, size_t *blob_sz); -struct ST_hyperloglog *ST_hyperloglog_deserialize(const char *blob, size_t blob_sz); -int ST_hyperloglog_merge(struct ST_hyperloglog *dst, const struct ST_hyperloglog *src); -void ST_hyperloglog_merge_blob(struct ST_hyperloglog *dst, const char *blob, size_t blob_sz); -double ST_hyperloglog_error_for_precision(unsigned char precision); -void ST_hyperloglog_configure(struct ST_hyperloglog *h, unsigned char precision, int time_window_seconds, const struct timeval now); +int hyperloglog_add(struct hyperloglog *h, const char *key, size_t keylen); +int hyperloglog_add_hash(struct hyperloglog *h, uint64_t hash); +void hyperloglog_reset(struct hyperloglog *h); +double hyperloglog_count(const struct hyperloglog *h); +size_t hyperloglog_serialized_size(const struct hyperloglog *h); +void hyperloglog_serialize(const struct hyperloglog *h, char **blob, size_t *blob_sz); +void hyperloglog_serialize_into_base64(const struct hyperloglog *h, char **blob, size_t *blob_sz); +struct hyperloglog *hyperloglog_deserialize(const char *blob, size_t blob_sz); +int hyperloglog_merge(struct hyperloglog *dest, const struct hyperloglog *src); +void hyperloglog_merge_blob(struct hyperloglog *dest, const char *blob, size_t blob_sz); +double hyperloglog_error_for_precision(unsigned char precision); #ifdef __cplusplus } #endif
\ No newline at end of file diff --git a/src/metrics/metric.c b/src/metrics/metric.c index fc7ba28..95dc646 100644 --- a/src/metrics/metric.c +++ b/src/metrics/metric.c @@ -3,138 +3,73 @@ #include <stdlib.h> #include <assert.h> #include <string.h> -#include <stdarg.h> #include <limits.h> - -#include "uthash.h" #include "histogram_encoder.h" #include "serializer.h" #include "metric.h" -#include "fieldstat.h" -#include "tags/cell_manager.h" #include "hdr/hdr_histogram.h" #include "hdr/hdr_histogram_log.h" -#include "st_hyperloglog.h" - -#define MAX_N_DATA_ARRAY_ITEM_INIT 1024 +#include "hyperloglog.h" +#include "metric_manifest.h" struct metric_counter_or_gauge { long long value; }; -struct para_hdr{ - long long lowest_trackable_value; - long long highest_trackable_value; - int significant_figures; -}; - -struct para_hll { - char precision; -}; - -struct metric_parameter { - union { - struct para_hdr hdr; - struct para_hll hll; - }; -}; - -struct metric_info { - int id; - char *name; - struct metric_parameter *paras; - enum metric_type type; -}; - -typedef struct metric_measure_data *(*metric_func_new)(const struct metric_parameter *); -typedef void (*metric_func_del)(struct metric_measure_data *); -typedef int (*metric_func_merge)(struct metric_measure_data *, const struct metric_measure_data *); -typedef int (*metric_func_copy)(struct metric_measure_data *, const struct metric_measure_data *); -typedef void (*metric_func_serialize)(const struct metric_measure_data *, char **/*blob*/, size_t */*blob_size*/); -typedef struct metric_measure_data * (*metric_func_deserialize)(const char *, size_t ); +typedef struct metric_data *(*metric_func_new)(const union metric_parameter *); +typedef void (*metric_func_free)(struct metric_data *); +typedef int (*metric_func_merge)(struct metric_data *, const struct metric_data *); +typedef struct metric_data * (*metric_func_copy)(const struct metric_data *); +typedef void (*metric_func_serialize)(const struct metric_data *, char **/*blob*/, size_t */*blob_size*/); +// typedef struct metric_data * (*metric_func_deserialize)(const char *, size_t ); +typedef void (*metric_func_reset)(struct metric_data *); struct metric_scheme { metric_func_new new; - metric_func_del del; + metric_func_free free; metric_func_merge merge; metric_func_copy copy; metric_func_serialize serialize; - metric_func_deserialize deserialize; + // metric_func_deserialize deserialize; + metric_func_reset reset; }; -struct metric_measure_data { +struct metric_data { union { struct metric_counter_or_gauge *counter; - struct ST_hyperloglog *hll; + struct hyperloglog *hll; struct hdr_histogram *hdr; }; }; struct metric { - const struct metric_scheme *scheme; - struct metric_info *info; - struct metric_measure_data **data_array; - size_t max_n_array_item; - size_t n_array_item; + enum metric_type type; + struct metric_data *data; + bool operated_after_reset; }; - -/* -------------------------------------------------------------------------- */ -/* id cell map */ -/* -------------------------------------------------------------------------- */ -static inline struct metric_measure_data *metric_find_one_cell(const struct metric *pthis, int id) { - if (id >= pthis->max_n_array_item) { - return NULL; - } - return pthis->data_array[id]; -} - -void metric_add_one_cell(struct metric *pthis, int id, struct metric_measure_data *data) { - if (id >= pthis->max_n_array_item) { - while (id >= pthis->max_n_array_item) { - pthis->max_n_array_item *= 2; - } - pthis->data_array = (struct metric_measure_data **)realloc(pthis->data_array, pthis->max_n_array_item * sizeof(struct metric_measure_data *)); - memset(pthis->data_array + pthis->n_array_item, 0, (pthis->max_n_array_item - pthis->n_array_item) * sizeof(struct metric_measure_data *)); - } - assert(pthis->data_array[id] == NULL); - pthis->data_array[id] = data; - - if (id + 1 > pthis->n_array_item) { - pthis->n_array_item = id + 1; - } - return; -} - /* -------------------------------------------------------------------------- */ /* scheme */ /* -------------------------------------------------------------------------- */ /* --------------------------------- counter -------------------------------- */ -struct metric_measure_data *metric_scheme_counter_new(const struct metric_parameter *_para) +struct metric_data *metric_scheme_counter_new(const union metric_parameter *_para) { - struct metric_measure_data *data = (struct metric_measure_data *)malloc(sizeof(struct metric_measure_data)); + struct metric_data *data = (struct metric_data *)malloc(sizeof(struct metric_data)); struct metric_counter_or_gauge *counter = (struct metric_counter_or_gauge *)malloc(sizeof(struct metric_counter_or_gauge)); data->counter = counter; counter->value = 0; return data; } -static void metric_scheme_counter_free(struct metric_measure_data *data) +static void metric_scheme_counter_free(struct metric_data *data) { free(data->counter); free(data); } -/* - "counter": { - "value": 0 - "mode": 0(sum) / 1(max) / 2(min) - } -value: long long | mode: uint32 -*/ -static void metric_scheme_counter_serialize(const struct metric_measure_data *data, char **blob, size_t *blob_size) +static void metric_scheme_counter_serialize(const struct metric_data *data, char **blob, size_t *blob_size) { const struct metric_counter_or_gauge *counter = data->counter; struct fs_reader *reader = fs_reader_new(); @@ -144,7 +79,7 @@ static void metric_scheme_counter_serialize(const struct metric_measure_data *da fs_reader_finalize(reader, blob, blob_size); } -static int metric_scheme_counter_merge(struct metric_measure_data *pthis, const struct metric_measure_data *from) +static int metric_scheme_counter_merge(struct metric_data *pthis, const struct metric_data *from) { struct metric_counter_or_gauge *counter = pthis->counter; const struct metric_counter_or_gauge *from_counter = from->counter; @@ -153,18 +88,20 @@ static int metric_scheme_counter_merge(struct metric_measure_data *pthis, const return 0; } -static int metric_scheme_counter_copy(struct metric_measure_data *pthis, const struct metric_measure_data *from) +static struct metric_data *metric_scheme_counter_copy(const struct metric_data *from) { - struct metric_counter_or_gauge *counter = pthis->counter; - const struct metric_counter_or_gauge *from_counter = from->counter; - counter->value = from_counter->value; - return 0; + struct metric_data *pthis = (struct metric_data *)malloc(sizeof(struct metric_data)); + struct metric_counter_or_gauge *counter = (struct metric_counter_or_gauge *)malloc(sizeof(struct metric_counter_or_gauge)); + pthis->counter = counter; + + counter->value = from->counter->value; + return pthis; } -struct metric_measure_data *metric_scheme_counter_deserialize(const char *blob, size_t blob_size) +struct metric_data *metric_scheme_counter_deserialize(const char *blob, size_t blob_size) { struct fs_writer *writer = fs_writer_new(blob, blob_size); - struct metric_measure_data *ret = (struct metric_measure_data *)malloc(sizeof(struct metric_measure_data)); + struct metric_data *ret = (struct metric_data *)malloc(sizeof(struct metric_data)); struct metric_counter_or_gauge *counter = (struct metric_counter_or_gauge *)malloc(sizeof(struct metric_counter_or_gauge)); ret->counter = counter; @@ -174,171 +111,158 @@ struct metric_measure_data *metric_scheme_counter_deserialize(const char *blob, return ret; } +void metric_scheme_counter_reset(struct metric_data *data) +{ + data->counter->value = 0; +} + /* --------------------------------- hll -------------------------------- */ -struct metric_measure_data *metric_scheme_hll_new(const struct metric_parameter *para) +struct metric_data *metric_scheme_hll_new(const union metric_parameter *para) { - struct metric_measure_data *data = (struct metric_measure_data *)malloc(sizeof(struct metric_measure_data)); - struct ST_hyperloglog *hll = ST_hyperloglog_new(para->hll.precision); + struct metric_data *data = (struct metric_data *)malloc(sizeof(struct metric_data)); + struct hyperloglog *hll = hyperloglog_new(para->hll.precision); data->hll = hll; return data; } -static void metric_scheme_hll_free(struct metric_measure_data *data) +static void metric_scheme_hll_free(struct metric_data *data) { - ST_hyperloglog_free(data->hll); + hyperloglog_free(data->hll); free(data); } -static void metric_scheme_hll_serialize(const struct metric_measure_data *data, char **blob, size_t *blob_size) +static void metric_scheme_hll_serialize(const struct metric_data *data, char **blob, size_t *blob_size) { - ST_hyperloglog_serialize(data->hll, blob, blob_size); + hyperloglog_serialize(data->hll, blob, blob_size); } -static int metric_scheme_hll_merge(struct metric_measure_data *pthis, const struct metric_measure_data *from) +static int metric_scheme_hll_merge(struct metric_data *pthis, const struct metric_data *from) { - return ST_hyperloglog_merge(pthis->hll, from->hll); + return hyperloglog_merge(pthis->hll, from->hll); } -static int metric_scheme_hll_copy(struct metric_measure_data *pthis, const struct metric_measure_data *from) +struct metric_data *metric_scheme_hll_copy(const struct metric_data *from) { - return ST_hyperloglog_merge(pthis->hll, from->hll); + struct metric_data *pthis = (struct metric_data *)malloc(sizeof(struct metric_data)); + struct hyperloglog *hll = hyperloglog_new(from->hll->cfg.precision); + pthis->hll = hll; + hyperloglog_merge(hll, from->hll); + return pthis; } -struct metric_measure_data *metric_scheme_hll_deserialize(const char *blob, size_t blob_size) +struct metric_data *metric_scheme_hll_deserialize(const char *blob, size_t blob_size) { - struct metric_measure_data *ret = (struct metric_measure_data *)malloc(sizeof(struct metric_measure_data)); - struct ST_hyperloglog *hll = ST_hyperloglog_deserialize(blob, blob_size); + struct metric_data *ret = (struct metric_data *)malloc(sizeof(struct metric_data)); + struct hyperloglog *hll = hyperloglog_deserialize(blob, blob_size); ret->hll = hll; return ret; } +void metric_scheme_hll_reset(struct metric_data *data) +{ + hyperloglog_reset(data->hll); +} /* --------------------------------- histogram -------------------------------- */ -struct metric_measure_data *metric_scheme_histogram_new(const struct metric_parameter *para) +struct metric_data *metric_scheme_histogram_new(const union metric_parameter *para) { struct hdr_histogram* tmp_p_hdr = NULL; int ret = hdr_init((int64_t)para->hdr.lowest_trackable_value, (int64_t)para->hdr.highest_trackable_value, para->hdr.significant_figures, &tmp_p_hdr); if (ret != 0) { return NULL; } - struct metric_measure_data *data = (struct metric_measure_data *)malloc(sizeof(struct metric_measure_data)); + struct metric_data *data = (struct metric_data *)malloc(sizeof(struct metric_data)); data->hdr = tmp_p_hdr; return data; } -static void metric_scheme_histogram_free(struct metric_measure_data *data) +static void metric_scheme_histogram_free(struct metric_data *data) { hdr_close(data->hdr); free(data); } -static void metric_scheme_histogram_serialize(const struct metric_measure_data *data, char **blob, size_t *blob_size) +static void metric_scheme_histogram_serialize(const struct metric_data *data, char **blob, size_t *blob_size) { histogram_encode_into_blob(data->hdr, blob, blob_size); } -static int metric_scheme_histogram_merge(struct metric_measure_data *pthis, const struct metric_measure_data *from) +static int metric_scheme_histogram_merge(struct metric_data *pthis, const struct metric_data *from) { (void)hdr_add(pthis->hdr, from->hdr); return 0; } -static int metric_scheme_histogram_copy(struct metric_measure_data *pthis, const struct metric_measure_data *from) +static struct metric_data *metric_scheme_histogram_copy(const struct metric_data *from) { - (void)hdr_add(pthis->hdr, from->hdr); - return 0; + struct hdr_histogram* tmp_p_hdr = NULL; + hdr_init(from->hdr->lowest_discernible_value, from->hdr->highest_trackable_value, from->hdr->significant_figures, &tmp_p_hdr); + + struct metric_data *data = (struct metric_data *)malloc(sizeof(struct metric_data)); + data->hdr = tmp_p_hdr; + + hdr_add(data->hdr, from->hdr); + return data; } -struct metric_measure_data *metric_scheme_histogram_deserialize(const char *blob, size_t blob_size) +struct metric_data *metric_scheme_histogram_deserialize(const char *blob, size_t blob_size) { struct hdr_histogram *hdr = histogram_decode_from_blob(blob, blob_size); - struct metric_measure_data *data = (struct metric_measure_data *)malloc(sizeof(struct metric_measure_data)); + struct metric_data *data = (struct metric_data *)malloc(sizeof(struct metric_data)); data->hdr = hdr; return data; } +void metric_scheme_histogram_reset(struct metric_data *data) +{ + hdr_reset(data->hdr); +} + static const struct metric_scheme g_metric_scheme_table[] = { [METRIC_TYPE_COUNTER] = { .new = metric_scheme_counter_new, - .del = metric_scheme_counter_free, + .free = metric_scheme_counter_free, .serialize = metric_scheme_counter_serialize, - .deserialize = metric_scheme_counter_deserialize, + // .deserialize = metric_scheme_counter_deserialize, .merge = metric_scheme_counter_merge, .copy = metric_scheme_counter_copy, + .reset = metric_scheme_counter_reset, }, [METRIC_TYPE_HLL] = { .new = metric_scheme_hll_new, - .del = metric_scheme_hll_free, + .free = metric_scheme_hll_free, .serialize = metric_scheme_hll_serialize, - .deserialize = metric_scheme_hll_deserialize, + // .deserialize = metric_scheme_hll_deserialize, .merge = metric_scheme_hll_merge, .copy = metric_scheme_hll_copy, + .reset = metric_scheme_hll_reset, }, [METRIC_TYPE_HISTOGRAM] = { .new = metric_scheme_histogram_new, - .del = metric_scheme_histogram_free, + .free = metric_scheme_histogram_free, .serialize = metric_scheme_histogram_serialize, - .deserialize = metric_scheme_histogram_deserialize, + // .deserialize = metric_scheme_histogram_deserialize, .merge = metric_scheme_histogram_merge, .copy = metric_scheme_histogram_copy, + .reset = metric_scheme_histogram_reset, }, }; + /* -------------------------------------------------------------------------- */ /* metric */ /* -------------------------------------------------------------------------- */ -struct metric_parameter *construct_parameters(enum metric_type type, ...) -{ - struct metric_parameter *paras = (struct metric_parameter *)malloc(sizeof(struct metric_parameter)); - va_list ap; - va_start(ap, type); - switch (type) { - case METRIC_TYPE_COUNTER: - break; - case METRIC_TYPE_HLL: - paras->hll.precision = (char)va_arg(ap, int); - break; - case METRIC_TYPE_HISTOGRAM: - paras->hdr.lowest_trackable_value = va_arg(ap, long long); - paras->hdr.highest_trackable_value = va_arg(ap, long long); - paras->hdr.significant_figures = va_arg(ap, int); - break; - default: - assert(0); - } - va_end(ap); - return paras; -} - -struct metric_info *metric_info_new(const char *name, enum metric_type type, struct metric_parameter *para) -{ - struct metric_info *info = (struct metric_info *)malloc(sizeof(struct metric_info)); - info->name = strdup(name); - info->paras = para; - info->type = type; - info->id = -1; - return info; -} - -void metric_info_free(struct metric_info *info) -{ - free(info->name); - free(info->paras); - free(info); -} -struct metric *metric_new(const char *name, enum metric_type type, struct metric_parameter *para) +struct metric *metric_new(const struct metric_manifest *manifest) { struct metric *pthis = (struct metric *)calloc(1, sizeof(struct metric)); - pthis->info = metric_info_new(name, type, para); - pthis->scheme = &g_metric_scheme_table[type]; - pthis->max_n_array_item = MAX_N_DATA_ARRAY_ITEM_INIT; - pthis->data_array = (struct metric_measure_data **)calloc(MAX_N_DATA_ARRAY_ITEM_INIT, sizeof(struct metric_measure_data *)); + pthis->data = g_metric_scheme_table[manifest->type].new(manifest->parameters); + pthis->type = manifest->type; return pthis; } @@ -348,498 +272,106 @@ void metric_free(struct metric *pthis) if (pthis == NULL) { return; } - metric_info_free(pthis->info); - for (size_t i = 0; i < pthis->n_array_item; i++) { - if (pthis->data_array[i] == NULL) { - continue; - } - pthis->scheme->del(pthis->data_array[i]); - } - free(pthis->data_array); + g_metric_scheme_table[pthis->type].free(pthis->data); free(pthis); } void metric_reset(struct metric *pthis) { - for (int i = 0; i < pthis->n_array_item; i++) { - if (pthis->data_array[i] == NULL) { - continue; - } - pthis->scheme->del(pthis->data_array[i]); - } - pthis->n_array_item = 0; - memset(pthis->data_array, 0, sizeof(struct metric_measure_data *) * pthis->max_n_array_item); -} - -void metric_info_copy(struct metric_info *dst, const struct metric_info *src) -{ - dst->name = strdup(src->name); - if (src->paras == NULL) { - dst->paras = NULL; - } else { - dst->paras = (struct metric_parameter *)malloc(sizeof(struct metric_parameter)); - memcpy(dst->paras, src->paras, sizeof(struct metric_parameter)); - } - - dst->type = src->type; - dst->id = src->id; -} - -struct metric *metric_fork(const struct metric *src) -{ - struct metric *dst = (struct metric *)malloc(sizeof(struct metric)); - dst->info = (struct metric_info *)malloc(sizeof(struct metric_info)); - metric_info_copy(dst->info, src->info); - dst->scheme = &g_metric_scheme_table[src->info->type]; - - dst->n_array_item = 0; - dst->max_n_array_item = src->max_n_array_item; - dst->data_array = (struct metric_measure_data **)calloc(src->max_n_array_item, sizeof(struct metric_measure_data *)); - - return dst; -} - -struct metric *metric_copy(const struct metric *src) -{ - struct metric *dst = metric_fork(src); - dst->n_array_item = src->n_array_item; - - for (int i = 0; i < src->n_array_item; i++) { - if (src->data_array[i] == NULL) { - continue; - } - struct metric_measure_data *dest_data = src->scheme->new(src->info->paras); - src->scheme->copy(dest_data, src->data_array[i]); - dst->data_array[i] = dest_data; - } - return dst; -} - -/* - ----------------------------------- counter ---------------------------------- -{ - "type": 0 - "merge_type" -} - ----------------------------------- hll ---------------------------------- -{ - "type": 1, - "prec": <precision value> -} - ----------------------------------- histogram ---------------------------------- -{ - "type": 2, - "min": <lowest_trackable_value> - "max": <highest_trackable_value> - "prec": <significant_figures> - -type:uint32 | min: long long | max: long long | prec: uint32 -} -*/ -void metric_serialize_parameters(const struct metric *metric, char **blob, size_t *blob_size) -{ - struct metric_parameter *para = metric->info->paras; - struct fs_reader *reader = fs_reader_new(); - - fs_reader_read_uint(reader, metric->info->type); - - switch (metric->info->type) - { - case METRIC_TYPE_COUNTER: - break; - case METRIC_TYPE_HLL: - fs_reader_read_uint(reader, para->hll.precision); - break; - case METRIC_TYPE_HISTOGRAM: - fs_reader_read_longlong(reader, para->hdr.lowest_trackable_value); - fs_reader_read_longlong(reader, para->hdr.highest_trackable_value); - fs_reader_read_uint(reader, para->hdr.significant_figures); - break; - default: - break; - } - - fs_reader_finalize(reader, blob, blob_size); -} - -struct metric_parameter *deserialize_parameters(const char *blob, size_t blob_size) -{ - struct fs_writer *writer = fs_writer_new(blob, blob_size); - - enum metric_type type; - fs_writer_write_uint(writer, (uint32_t *)&type); - - struct metric_parameter *field_para = NULL; - switch (type) { - case METRIC_TYPE_COUNTER: - - break; - case METRIC_TYPE_HLL: { - int precision; - fs_writer_write_uint(writer, (uint32_t *)&precision); - field_para = construct_parameters(type, precision); - } - break; - case METRIC_TYPE_HISTOGRAM: { - long long min; - long long max; - int prec; - fs_writer_write_longlong(writer, &min); - fs_writer_write_longlong(writer, &max); - fs_writer_write_uint(writer, (uint32_t *)&prec); - - field_para = construct_parameters(type, min, max, prec); - } - break; - default: - break; - } - - fs_writer_free(writer); - return field_para; -} - -/* -{ - "id":<cell_id>, - "PL":<content_blob> - -id:uint | PL:bin -} -*/ -void blob_wrap_basic_info(int cell_id, const char *blob, size_t blob_size, char **new_blob, size_t *new_blob_size) -{ - struct fs_reader *reader = fs_reader_new(); - - fs_reader_read_uint(reader, cell_id); - fs_reader_read_bin(reader, blob, blob_size); - - fs_reader_finalize(reader, new_blob, new_blob_size); -} - -// user must free new_blob -void blob_unpack_basic_info(const char *blob, size_t blob_size, char **new_blob, size_t *new_blob_size, int *cell_id) -{ - struct fs_writer *writer = fs_writer_new(blob, blob_size); - - fs_writer_write_uint(writer, (uint32_t *)cell_id); - fs_writer_write_bin(writer, new_blob, new_blob_size); - - fs_writer_free(writer); -} - -/* -metric blob: - { - "name":<metric_name> - "type":<field_type> - "id":<field_id> - "para":[may be null, one number, or a double array] - "data":[{ - "id":<cell_id> - "PL":<content_blob(gen by scheme)> - }] -} - -name:string | type:uint | para:bin | data_id:int_array | data_payload:bin_array -*/ -int metric_serialize(const struct metric *pthis, char **blob, size_t *blob_size) -{ - struct fs_reader *reader = fs_reader_new(); - - fs_reader_read_str(reader, pthis->info->name); - fs_reader_read_uint(reader, pthis->info->type); - - char *para_blob = NULL; - size_t para_blob_size = 0; - metric_serialize_parameters(pthis, ¶_blob, ¶_blob_size); - fs_reader_read_bin(reader, para_blob, para_blob_size); - free(para_blob); - - int *cell_ids = NULL; - size_t cell_ids_size = 0; - metric_get_cell_ids(pthis, &cell_ids, &cell_ids_size); - if (cell_ids == NULL) { - fs_reader_read_nil(reader); - } else { - fs_reader_start_bin_array(reader, cell_ids_size); - for (int i = 0; i < cell_ids_size; i++) { - struct metric_measure_data *data = metric_find_one_cell(pthis, cell_ids[i]); - - char *data_blob = NULL; - size_t data_blob_size = 0; - pthis->scheme->serialize(data, &data_blob, &data_blob_size); - - char *tmp_data_wrapped_blob = NULL; - size_t tmp_data_wrapped_blob_size = 0; - blob_wrap_basic_info(cell_ids[i], data_blob, data_blob_size, &tmp_data_wrapped_blob, &tmp_data_wrapped_blob_size); - fs_reader_read_bin(reader, tmp_data_wrapped_blob, tmp_data_wrapped_blob_size); - free(data_blob); - free(tmp_data_wrapped_blob); - } - - free(cell_ids); - } - - fs_reader_finalize(reader, blob, blob_size); - - return 0; + g_metric_scheme_table[pthis->type].reset(pthis->data); + pthis->operated_after_reset = false; } -struct metric *metric_deserialize(const char *blob, size_t blob_size) -{ - struct fs_writer *writer = fs_writer_new(blob, blob_size); +struct metric *metric_copy(const struct metric *src) { + struct metric *dest = (struct metric *)malloc(sizeof(struct metric)); + dest->type = src->type; + dest->data = g_metric_scheme_table[dest->type].copy(src->data); + dest->operated_after_reset = src->operated_after_reset; - char *metric_name; - fs_writer_write_str(writer, &metric_name); - enum metric_type type; - fs_writer_write_uint(writer, (uint32_t *)&type); - - char *para_blob; - size_t para_blob_size; - fs_writer_write_bin(writer, ¶_blob, ¶_blob_size); - struct metric_parameter *metric_para = deserialize_parameters(para_blob, para_blob_size); - struct metric *metric_out = metric_new(metric_name, type, metric_para); - free(metric_name); - free(para_blob); - - int ret = fs_writer_expect_nil(writer); - assert(ret >= 0); - if (ret == 1) { // no cells in this metric - fs_writer_free(writer); - return metric_out; - } - - char **data_blob_raw = NULL; - size_t *data_blob_raw_size = NULL; - size_t n_data_blob_raw = 0; - fs_writer_write_bin_array(writer, &data_blob_raw, &n_data_blob_raw, &data_blob_raw_size); - - for (int i = 0; i < n_data_blob_raw; i++) { - int cell_id; - char *data_blob = NULL; - size_t data_blob_size = 0; - blob_unpack_basic_info(data_blob_raw[i], data_blob_raw_size[i], &data_blob, &data_blob_size, &cell_id); - struct metric_measure_data *data = metric_out->scheme->deserialize(data_blob, data_blob_size); - free(data_blob); - metric_add_one_cell(metric_out, cell_id, data); - } - - for (int i = 0; i < n_data_blob_raw; i++) { - free(data_blob_raw[i]); - } - free(data_blob_raw); - free(data_blob_raw_size); - fs_writer_free(writer); - - return metric_out; + return dest; } -int metric_merge_or_copy_cell(struct metric *dest, const struct metric *src, int dest_cell_id, int src_cell_id) -{ - struct metric_measure_data *src_data = metric_find_one_cell(src, src_cell_id); - if (src_data == NULL) { - return 1; - } - - struct metric_measure_data *dest_data = metric_find_one_cell(dest, dest_cell_id); - if (dest_data == NULL) { // copy - dest_data = src->scheme->new(src->info->paras); - int ret = dest->scheme->copy(dest_data, src_data); - if (ret == -1) { - dest->scheme->del(dest_data); - return -1; - } - metric_add_one_cell(dest, dest_cell_id, dest_data); - return 0; +//return -1 when merge error. 0 when success. 1 when src has no cell. +int metric_merge(struct metric *dest, const struct metric *src) { + if (src->operated_after_reset) { + dest->operated_after_reset = true; } - - //merge - return dest->scheme->merge(dest_data, src_data); + return g_metric_scheme_table[dest->type].merge(dest->data, src->data); } -void metric_get_plain_blob(const struct metric *pthis, int cell_id, char **blob, size_t *blob_size) -{ - struct metric_measure_data *data = metric_find_one_cell(pthis, cell_id); - if (data == NULL) { - *blob = NULL; - *blob_size = 0; +void metric_serialize(const struct metric *pthis, char **blob, size_t *blob_size) { + struct metric_data *data = pthis->data; + enum metric_type type = pthis->type; + if (type == METRIC_TYPE_HLL) { + hyperloglog_serialize_into_base64(data->hll, blob, blob_size); return; } - if (pthis->info->type == METRIC_TYPE_HLL) { - ST_hyperloglog_serialize_for_networking(data->hll, blob, blob_size); - return; - } - if (pthis->info->type == METRIC_TYPE_HISTOGRAM) { + if (type == METRIC_TYPE_HISTOGRAM) { histogram_encode_into_b64(data->hdr, blob, blob_size); return; } - pthis->scheme->serialize(data, blob, blob_size); + g_metric_scheme_table[type].serialize(data, blob, blob_size); } -void metric_delete_cell(struct metric *pthis, int cell_id) -{ - struct metric_measure_data *data = metric_find_one_cell(pthis, cell_id); - if (data == NULL) { - return; - } - pthis->scheme->del(data); - pthis->data_array[cell_id] = NULL; +bool metric_check_if_cleared(const struct metric *pthis) { + return !pthis->operated_after_reset; } /* -------------------------------------------------------------------------- */ -/* metric operations */ +/* specific metric operations */ /* -------------------------------------------------------------------------- */ -static inline struct metric_measure_data *metric_find_or_new_cell(struct metric *pthis, int cell_id) -{ - struct metric_measure_data *data = metric_find_one_cell(pthis, cell_id); - - if (data == NULL) { - data = pthis->scheme->new(pthis->info->paras); - metric_add_one_cell(pthis, cell_id, data); - } - return data; -} +void metric_counter_incrby(struct metric *pthis, long long value) { + struct metric_counter_or_gauge *counter = pthis->data->counter; + counter->value += value; -struct metric *metric_counter_new(const char *name) -{ - struct metric_parameter *metric_para = NULL; - struct metric *pthis = metric_new(name, METRIC_TYPE_COUNTER, metric_para); - - return pthis; + pthis->operated_after_reset = true; } -void metric_counter_incrby(struct metric *pthis, int cell_id, long long value) -{ - struct metric_measure_data *data = metric_find_or_new_cell(pthis, cell_id); - data->counter->value += value; -} - -int metric_counter_set(struct metric *pthis, int cell_id, long long value) -{ - struct metric_measure_data *data = metric_find_or_new_cell(pthis, cell_id); - data->counter->value = value; +void metric_counter_set(struct metric *pthis, long long value) { + struct metric_counter_or_gauge *counter = pthis->data->counter; + counter->value = value; - return 0; + pthis->operated_after_reset = true; } -int metric_counter_get(const struct metric *pthis, int cell_id, long long *value_out) -{ - const struct metric_measure_data *data = metric_find_one_cell(pthis, cell_id); - if (data == NULL) { - *value_out = 0; - return -1; - } - const struct metric_counter_or_gauge *counter = data->counter; - *value_out = counter->value; - return 0; +long long metric_counter_get(const struct metric *pthis) { + return pthis->data->counter->value; } -struct metric *metric_hll_new(const char *name, unsigned char precision) -{ - struct metric_parameter *metric_para = construct_parameters(METRIC_TYPE_HLL, precision); - struct metric *pthis = metric_new(name, METRIC_TYPE_HLL, metric_para); +void metric_hll_add(struct metric *pthis, const char *key, size_t key_len) { + hyperloglog_add(pthis->data->hll, key, key_len); - return pthis; + pthis->operated_after_reset = true; } -void metric_hll_add(struct metric *pthis, int cell_id, const char *key, size_t key_len) -{ - struct metric_measure_data *data = metric_find_or_new_cell(pthis, cell_id); +void metric_hll_add_hash(struct metric *pthis, unsigned long long hash) { + hyperloglog_add_hash(pthis->data->hll, hash); - ST_hyperloglog_add(data->hll, key, key_len); + pthis->operated_after_reset = true; } -double metric_hll_get(const struct metric *pthis, int cell_id) -{ - const struct metric_measure_data *data = metric_find_one_cell(pthis, cell_id); - if (data == NULL) { - return -1; - } - return ST_hyperloglog_count(data->hll); +double metric_hll_get(const struct metric *pthis) { + return hyperloglog_count(pthis->data->hll); } -struct metric *metric_histogram_new(const char *name, long long lowest_trackable_value, long long highest_trackable_value, int significant_figures) -{ - struct metric_parameter *metric_para = construct_parameters(METRIC_TYPE_HISTOGRAM, lowest_trackable_value, highest_trackable_value, significant_figures); - struct metric *pthis = metric_new(name, METRIC_TYPE_HISTOGRAM, metric_para); - - return pthis; -} - -int metric_histogram_record(struct metric *pthis, int cell_id, long long value) -{ - struct metric_measure_data *data = metric_find_or_new_cell(pthis, cell_id); - if (value > data->hdr->highest_trackable_value) { - value = data->hdr->highest_trackable_value; +int metric_histogram_record(struct metric *pthis, long long value) { + if (value > pthis->data->hdr->highest_trackable_value) { + value = pthis->data->hdr->highest_trackable_value; } - - bool ret = hdr_record_value(data->hdr, value); + bool ret = hdr_record_value(pthis->data->hdr, value); if (!ret) { return -1; } + + pthis->operated_after_reset = true; return 0; } -long long metric_histogram_value_at_percentile(const struct metric *pthis, int cell_id, double percentile) -{ - const struct metric_measure_data *data = metric_find_one_cell(pthis, cell_id); - if (data == NULL) { - return -1; - } - return hdr_value_at_percentile(data->hdr, percentile); +long long metric_histogram_value_at_percentile(const struct metric *pthis, double percentile) { + return hdr_value_at_percentile(pthis->data->hdr, percentile); } -long long metric_histogram_count_le_value(const struct metric *pthis, int cell_id, long long value) -{ - const struct metric_measure_data *data = metric_find_one_cell(pthis, cell_id); - if (data == NULL) { - return -1; - } - return hdr_count_le_value(data->hdr, value); +long long metric_histogram_count_le_value(const struct metric *pthis, long long value) { + return hdr_count_le_value(pthis->data->hdr, value); } - -/* -------------------------------------------------------------------------- */ -/* query on the metric */ -/* -------------------------------------------------------------------------- */ -void metric_get_cell_ids(const struct metric *pthis, int **cell_ids, size_t *cell_id_count) -{ - size_t n_id_max = pthis->n_array_item; - if (n_id_max == 0) { - *cell_ids = NULL; - *cell_id_count = 0; - return; - } - - int *tmp_id_container = (int *)malloc(sizeof(int) * n_id_max); - int n_id = 0; - for (int i = 0; i < n_id_max; i++) { - if (pthis->data_array[i] != NULL) { - tmp_id_container[n_id++] = i; - } - } - - if (n_id == 0) { - *cell_ids = NULL; - *cell_id_count = 0; - free(tmp_id_container); - return; - } - - *cell_id_count = n_id; - *cell_ids = tmp_id_container; -} - -const char *metric_get_name(const struct metric *metrics) -{ - return metrics->info->name; -} - -enum metric_type metric_get_type(const struct metric *metrics) -{ - return metrics->info->type; -}
\ No newline at end of file diff --git a/src/metrics/metric.h b/src/metrics/metric.h index 845a792..0dee126 100644 --- a/src/metrics/metric.h +++ b/src/metrics/metric.h @@ -2,38 +2,29 @@ #include <stddef.h> #include <stdbool.h> -#include "fieldstat.h" +#include "fieldstat.h" // for enum metric_type +#include "metric_manifest.h" struct metric; -const char *metric_get_name(const struct metric *metrics); // don't free the return value -enum metric_type metric_get_type(const struct metric *metrics); void metric_free(struct metric *pthis); void metric_reset(struct metric *pthis); struct metric *metric_copy(const struct metric *src); -struct metric *metric_fork(const struct metric *src); -int metric_serialize(const struct metric *pthis, char **blob, size_t *blob_size); -struct metric *metric_deserialize(const char *blob, size_t blob_size); +struct metric *metric_new(const struct metric_manifest *manifest); +int metric_merge(struct metric *dest, const struct metric *src); +void metric_serialize(const struct metric *pthis, char **blob, size_t *blob_size); +bool metric_check_if_cleared(const struct metric *pthis); -//return -1 when merge error. 0 when success. 1 when src has no cell. -int metric_merge_or_copy_cell(struct metric *dest, const struct metric *src, int dest_cell_id, int src_cell_id); -void metric_get_plain_blob(const struct metric *pthis, int cell_id, char **blob, size_t *blob_size); -void metric_delete_cell(struct metric *pthis, int cell_id); +void metric_counter_incrby(struct metric *pthis, long long value); +void metric_counter_set(struct metric *pthis, long long value); +long long metric_counter_get(const struct metric *pthis); -struct metric *metric_counter_new(const char *name); -void metric_counter_incrby(struct metric *pthis, int cell_id, long long value); -int metric_counter_set(struct metric *pthis, int cell_id, long long value); -int metric_counter_get(const struct metric *pthis, int cell_id, long long *value_out); +void metric_hll_add(struct metric *pthis, const char *key, size_t key_len); +void metric_hll_add_hash(struct metric *pthis, unsigned long long hash); +double metric_hll_get(const struct metric *pthis); -struct metric *metric_hll_new(const char *name, unsigned char precision); -void metric_hll_add(struct metric *pthis, int cell_id, const char *key, size_t key_len); -double metric_hll_get(const struct metric *pthis, int cell_id); - -struct metric *metric_histogram_new(const char *name, long long lowest_trackable_value, long long highest_trackable_value, int significant_figures); -int metric_histogram_record(struct metric *pthis, int cell_id, long long value); +int metric_histogram_record(struct metric *pthis, long long value); // for the meaning of these two query method, refer to https://prometheus.io/docs/concepts/metric_types/, and search for "histogram" and "summary" -long long metric_histogram_value_at_percentile(const struct metric *pthis, int cell_id, double percentile); -long long metric_histogram_count_le_value(const struct metric *pthis, int cell_id, long long value); - +long long metric_histogram_value_at_percentile(const struct metric *pthis, double percentile); +long long metric_histogram_count_le_value(const struct metric *pthis, long long value); -void metric_get_cell_ids(const struct metric *pthis, int **cell_ids, size_t *cell_id_count); diff --git a/src/metrics/metric_manifest.c b/src/metrics/metric_manifest.c new file mode 100644 index 0000000..fd03209 --- /dev/null +++ b/src/metrics/metric_manifest.c @@ -0,0 +1,134 @@ + +#include "metric_manifest.h" + +#define DEFAULT_N_METRIC 32 + +struct metric_manifest; // defined in metric_manifest.h + +struct metric_manifest_manager { + struct metric_manifest *hash_table; + + struct metric_manifest **manifests; + size_t manifest_size; + size_t manifest_cnt; +}; + +struct metric_manifest_manager *metric_manifest_manager_new() { + struct metric_manifest_manager *ret = calloc(1, sizeof(struct metric_manifest_manager)); + + ret->manifest_size = DEFAULT_N_METRIC; + ret->manifests = calloc(DEFAULT_N_METRIC, sizeof(struct metric_manifest *)); + ret->manifest_cnt = 0; + + return ret; +} + +void metric_manifest_manager_free(struct metric_manifest_manager *pthis) { + struct metric_manifest *cur, *tmp; + HASH_ITER(hh, pthis->hash_table, cur, tmp) { + HASH_DEL(pthis->hash_table, cur); + } + + for (size_t i = 0; i < pthis->manifest_cnt; i++) { + metric_manifest_free(pthis->manifests[i]); + } + free(pthis->manifests); + free(pthis); +} + +const struct metric_manifest* metric_manifest_manager_get_by_id(const struct metric_manifest_manager *pthis, int metric_id) { + if (metric_id >= pthis->manifest_cnt || metric_id < 0) { + return NULL; + } + return pthis->manifests[metric_id]; +} + +const struct metric_manifest* metric_manifest_manager_get_by_name(const struct metric_manifest_manager *pthis, const char *name) { + struct metric_manifest *manifest = NULL; + HASH_FIND_STR(pthis->hash_table, name, manifest); + return manifest; +} + +void add_manifest_to_position(struct metric_manifest_manager *pthis, struct metric_manifest *manifest, int metric_id) +{ + if (metric_id >= pthis->manifest_size) { + pthis->manifests = realloc(pthis->manifests, sizeof(struct metric_manifest *) * pthis->manifest_size * 2); + memset(pthis->manifests + pthis->manifest_size, 0, sizeof(struct metric_manifest *) * (pthis->manifest_size)); + pthis->manifest_size *= 2; + } + + pthis->manifests[metric_id] = manifest; + if (metric_id >= pthis->manifest_cnt) { + pthis->manifest_cnt = metric_id + 1; + } + + HASH_ADD_KEYPTR(hh, pthis->hash_table, manifest->name, strlen(manifest->name), manifest); +} + +int metric_manifest_manager_add(struct metric_manifest_manager *pthis, struct metric_manifest *manifest) { + struct metric_manifest *tmp; + HASH_FIND_STR(pthis->hash_table, manifest->name, tmp); + if (tmp != NULL) { + return -1; + } + + int id = pthis->manifest_cnt; + add_manifest_to_position(pthis, manifest, id); + + manifest->id = id; + return id; +} + +const struct metric_manifest **metric_manifest_manager_list(const struct metric_manifest_manager *pthis, size_t *n_manifest) { + *n_manifest = pthis->manifest_cnt; + return (const struct metric_manifest **)pthis->manifests; +} + +bool metric_manifest_manager_equal(const struct metric_manifest_manager *pthis, const struct metric_manifest_manager *other) { + if (pthis->manifest_cnt != other->manifest_cnt) { + return false; + } + + for (size_t i = 0; i < pthis->manifest_cnt; i++) { + if (pthis->manifests[i]->type != other->manifests[i]->type) { + return false; + } + if (strcmp(pthis->manifests[i]->name, other->manifests[i]->name) != 0) { + return false; + } + } + + return true; +} + +void metric_manifest_free(struct metric_manifest *manifest) { + free(manifest->name); + free(manifest->parameters); + free(manifest); +} + +struct metric_manifest *metric_manifest_copy(const struct metric_manifest *src) +{ + struct metric_manifest *dest = (struct metric_manifest *)malloc(sizeof(struct metric_manifest)); + dest->name = strdup(src->name); + if (src->parameters == NULL) { + dest->parameters = NULL; + } else { + dest->parameters = (union metric_parameter *)malloc(sizeof(union metric_parameter)); + memcpy(dest->parameters, src->parameters, sizeof(union metric_parameter)); + } + + dest->type = src->type; + dest->id = src->id; + + return dest; +} + +struct metric_manifest_manager *metric_manifest_manager_copy(const struct metric_manifest_manager *pthis) { + struct metric_manifest_manager *ret = metric_manifest_manager_new(); + for (size_t i = 0; i < pthis->manifest_cnt; i++) { + struct metric_manifest *manifest = metric_manifest_copy(pthis->manifests[i]); + metric_manifest_manager_add(ret, manifest); + } + return ret; +} diff --git a/src/metrics/metric_manifest.h b/src/metrics/metric_manifest.h new file mode 100644 index 0000000..632b314 --- /dev/null +++ b/src/metrics/metric_manifest.h @@ -0,0 +1,42 @@ +#pragma once + +#include "uthash.h" +#include "fieldstat.h" + +struct histogram_parameters { + long long lowest_trackable_value; + long long highest_trackable_value; + int significant_figures; +}; + +struct hll_parameters { + char precision; +}; + +union metric_parameter { + struct histogram_parameters hdr; + struct hll_parameters hll; +}; + +struct metric_manifest { + int id; + char *name; + union metric_parameter *parameters; + enum metric_type type; + + UT_hash_handle hh; +}; + +struct metric_manifest *metric_manifest_copy(const struct metric_manifest *src); +void metric_manifest_free(struct metric_manifest *pthis); + +struct metric_manifest_manager; + +struct metric_manifest_manager *metric_manifest_manager_new(); +void metric_manifest_manager_free(struct metric_manifest_manager *pthis); +int metric_manifest_manager_add(struct metric_manifest_manager *pthis, struct metric_manifest *manifest); +struct metric_manifest_manager *metric_manifest_manager_copy(const struct metric_manifest_manager *pthis); + +const struct metric_manifest **metric_manifest_manager_list(const struct metric_manifest_manager *pthis, size_t *n_manifest); +const struct metric_manifest*metric_manifest_manager_get_by_id(const struct metric_manifest_manager *pthis, int metric_id); +const struct metric_manifest*metric_manifest_manager_get_by_name(const struct metric_manifest_manager *pthis, const char *name);
\ No newline at end of file diff --git a/src/metrics/python_api.c b/src/metrics/python_api.c index c7c4c6d..3f52149 100644 --- a/src/metrics/python_api.c +++ b/src/metrics/python_api.c @@ -5,7 +5,7 @@ #include "histogram_encoder.h" #include "base64/b64.h" -#include "st_hyperloglog.h" +#include "hyperloglog.h" // user must free the buf after use void *fieldstat_histogram_base64_decode(char *buf) @@ -16,12 +16,12 @@ void *fieldstat_histogram_base64_decode(char *buf) return hdr; } -long long fieldstat_histogram_value_at_percentile(void* h, double percentile) +long long fieldstat_histogram_value_at_percentile_api(void* h, double percentile) { return hdr_value_at_percentile((const struct hdr_histogram *)h, percentile); } -long long fieldstat_histogram_count_le_value(void* h, long long value) +long long fieldstat_histogram_count_le_value_api(void* h, long long value) { return hdr_count_le_value((const struct hdr_histogram *)h, value); } @@ -75,9 +75,9 @@ void fieldstat_histogram_free(void *h) void *hll_base64_decode(char *buf); double fieldstat_hll_base64_to_count(char *buf) { - struct ST_hyperloglog *hll = hll_base64_decode(buf); - double count = ST_hyperloglog_count(hll); - ST_hyperloglog_free(hll); + struct hyperloglog *hll = hll_base64_decode(buf); + double count = hyperloglog_count(hll); + hyperloglog_free(hll); return count; } @@ -101,7 +101,7 @@ void *hll_base64_decode(char *buf) unsigned char precision; memcpy(&precision, dec + sizeof(unsigned char), sizeof(unsigned char)); - struct ST_hyperloglog *hll_from_blob = ST_hyperloglog_new(precision); + struct hyperloglog *hll_from_blob = hyperloglog_new(precision); int num_reg = NUM_REG(precision); int words = INT_CEIL(num_reg, REG_PER_WORD); @@ -120,7 +120,7 @@ void *hll_base64_decode(char *buf) void fieldstat_hll_free(void *hll) { - ST_hyperloglog_free((struct ST_hyperloglog *)hll); + hyperloglog_free((struct hyperloglog *)hll); } // cppcheck-suppress [constParameterPointer, unmatchedSuppression] diff --git a/src/tags/cell_manager.c b/src/tags/cell_manager.c deleted file mode 100644 index 67e0cbf..0000000 --- a/src/tags/cell_manager.c +++ /dev/null @@ -1,399 +0,0 @@ -#include <stdio.h> -#include <assert.h> -#include <stdlib.h> -#include <string.h> - -#include "fieldstat.h" -#include "tags/cell_manager.h" -#include "tags/heavy_keeper.h" -#include "tags/my_ut_hash_inner.h" -#include "tags/my_ut_hash.h" -#include "serializer.h" - - -struct tag_id_map { - struct tag_hash_key *tag; - int cell_id; - bool dying; - UT_hash_handle hh; -}; - -struct cell_manager { - union { - struct heavy_keeper *topk_tag_id_map; - struct tag_id_map *comprehensive_tag_id_map; - }; - struct tag_hash_key **id_tag_array; - int id_tag_array_len; - int max_cell_id; - - enum sampling_mode sampling_mode; - int max_cell_num; - int current_cell_num; - int next_cell_id; -}; - -struct cube_manager { - struct tag_id_map *head; -}; - -struct cell_manager *cell_manager_new_without_map(enum sampling_mode sampling_mode, int max_cell_num) -{ - struct cell_manager *cell_manager = (struct cell_manager *)calloc(1, sizeof(struct cell_manager)); - - cell_manager->sampling_mode = sampling_mode; - cell_manager->max_cell_num = max_cell_num; - cell_manager->next_cell_id = 0; - - cell_manager->id_tag_array_len = max_cell_num; - if (cell_manager->id_tag_array_len > 4096) { // support dynamic size cube (especially when max_max_num is unlimited) - cell_manager->id_tag_array_len = 4096; - } - cell_manager->id_tag_array = calloc(cell_manager->id_tag_array_len, sizeof(struct tag_hash_key *)); - cell_manager->max_cell_id = -1; - - return cell_manager; -} - -struct cell_manager *cell_manager_new(enum sampling_mode sampling_mode, int max_cell_num) -{ - struct cell_manager *cell_manager = cell_manager_new_without_map(sampling_mode, max_cell_num); - - if (sampling_mode == SAMPLING_MODE_TOPK) { - cell_manager->topk_tag_id_map = heavy_keeper_with_data_new(max_cell_num); - } else { - cell_manager->comprehensive_tag_id_map = NULL; - } - - return cell_manager; -} - -void tag_id_map_free(struct tag_id_map *head) -{ - struct tag_id_map *node = NULL; - struct tag_id_map *tmp = NULL; - HASH_ITER(hh, head, node, tmp) { - HASH_DEL(head, node); - tag_hash_key_free(node->tag); - free(node); - } -} - -void cell_manager_reset(struct cell_manager *pthis) -{ - pthis->next_cell_id = 0; - pthis->max_cell_id = -1; - pthis->current_cell_num = 0; - memset(pthis->id_tag_array, 0, sizeof(struct tag_hash_key *) * pthis->id_tag_array_len); - - if (pthis->sampling_mode == SAMPLING_MODE_TOPK) { - heavy_keeper_reset(pthis->topk_tag_id_map); - return; - } - - struct tag_id_map *node = NULL; - struct tag_id_map *tmp = NULL; - HASH_ITER(hh, pthis->comprehensive_tag_id_map, node, tmp) { - if (!node->dying) { - node->dying = true; - continue; - } - HASH_DEL(pthis->comprehensive_tag_id_map, node); - tag_hash_key_free(node->tag); - free(node); - } -} - -void cell_manager_free(struct cell_manager *pthis) -{ - free(pthis->id_tag_array); - - if (pthis->sampling_mode == SAMPLING_MODE_TOPK) { - heavy_keeper_free(pthis->topk_tag_id_map); - } else { - tag_id_map_free(pthis->comprehensive_tag_id_map); - } - - free(pthis); -} - -const struct tag_hash_key *cell_manager_get_tag_by_cell_id(const struct cell_manager *pthis, int cell_id) -{ - return pthis->id_tag_array[cell_id]; -} - -void cell_manager_id_tag_array_add(struct cell_manager *pthis, int cell_id, struct tag_hash_key *tag) -{ - if (cell_id >= pthis->id_tag_array_len) { - int new_len = pthis->id_tag_array_len; - while (cell_id >= new_len) { - new_len *= 2; - } - - struct tag_hash_key **new_array = (struct tag_hash_key **)calloc(new_len, sizeof(struct tag_hash_key *)); - assert(new_array != NULL); - memcpy(new_array, pthis->id_tag_array, sizeof(struct tag_hash_key *) * pthis->id_tag_array_len); - free(pthis->id_tag_array); - pthis->id_tag_array = new_array; - pthis->id_tag_array_len = new_len; - } - - pthis->id_tag_array[cell_id] = tag; - - if (cell_id > pthis->max_cell_id) { - pthis->max_cell_id = cell_id; - } - pthis->current_cell_num++; -} - -long long cell_manager_get_count_by_tag(const struct cell_manager *pthis, const struct tag_hash_key *tag) -{ - if (pthis->sampling_mode == SAMPLING_MODE_COMPREHENSIVE) { - return -1; - } - - unsigned long long int count = 0; - int ret = heavy_keeper_query_one(pthis->topk_tag_id_map, tag, &count, NULL); - if (ret == -1) { - return -1; - } - - return count; -} - -int cell_manager_add_cell(struct cell_manager *pthis, const struct tag_hash_key *tag) -{ - struct tag_id_map *node = NULL; - HASH_FIND_TAG(pthis->comprehensive_tag_id_map, tag, node); - if (node != NULL && !node->dying) { - return node->cell_id; - } - - if (pthis->current_cell_num >= pthis->max_cell_num) { - return -1; - } - - if (node != NULL) { - node->dying = false; - node->cell_id = pthis->next_cell_id; - cell_manager_id_tag_array_add(pthis, pthis->next_cell_id, node->tag); - return pthis->next_cell_id++; - } - - struct tag_hash_key *tag_copy = tag_hash_key_copy(tag); - node = (struct tag_id_map *)malloc(sizeof(struct tag_id_map)); - node->tag = tag_copy; - node->cell_id = pthis->next_cell_id; - node->dying = false; - HASH_ADD_TAG(pthis->comprehensive_tag_id_map, tag, node); - - cell_manager_id_tag_array_add(pthis, pthis->next_cell_id, tag_copy); - return pthis->next_cell_id++; -} - -int cell_manager_add_cell_topk(struct cell_manager *pthis, const struct tag_hash_key *tag, unsigned long long int count, int *popped_cell_id) -{ - int ret = heavy_keeper_add(pthis->topk_tag_id_map, tag, count, pthis->next_cell_id, popped_cell_id); - if (ret == -1) { // didn't add - *popped_cell_id = -1; - return -1; - } - - if (ret == 1) { // use the original id - int existing_cell_id = -1; - heavy_keeper_query_one(pthis->topk_tag_id_map, tag, NULL, &existing_cell_id); - - *popped_cell_id = -1; - return existing_cell_id; - } - - struct tag_hash_key *tag_ret = heavy_keeper_get_true_pst_tag(pthis->topk_tag_id_map, tag); - - /* -------------------------------- new cell added-------------------------------- */ -/* -To mitigate the risk of unbounded array size due to potentially infinite growth of cell IDs, a maximum limit is set for cell IDs in the metric. -This limit is configured as 8 times the value of K. Even in the worst-case scenario of malicious attacks, this prevents excessive memory usage. -Even though there is a small possibility of not counting certain flows due to the limit, it is expected that the unallocated IDs would correspond to mouse flows, - which would have minimal impact on accuracy. -The number 8 was determined through experiment, where multiple tests were conducted, randomly generating flows by which cells are determined. - The observed stable cell ID count did not exceed 3 times the value of K. Therefore, a conservative value of 8 was chosen.*/ - if (pthis->next_cell_id >= pthis->max_cell_num * 8) { // max int is much larger, no risk of overflow - return -1; - } - - if (*popped_cell_id != -1) { // popped, no need to free the tag - pthis->id_tag_array[*popped_cell_id] = NULL; - } - cell_manager_id_tag_array_add(pthis, pthis->next_cell_id, tag_ret); - return pthis->next_cell_id++; -} - -int cell_manager_delete_cell(struct cell_manager *pthis, const struct tag_hash_key *tag) -{ - if (pthis->sampling_mode == SAMPLING_MODE_TOPK) { - return -1; - } - - struct tag_id_map *node = NULL; - HASH_FIND_TAG(pthis->comprehensive_tag_id_map, tag, node); - if (node == NULL) { - return -1; - } - - int cell_id = node->cell_id; - - HASH_DEL(pthis->comprehensive_tag_id_map, node); - tag_hash_key_free(node->tag); - free(node); - pthis->id_tag_array[cell_id] = NULL; - - return cell_id; -} - -struct cell_manager *cell_manager_copy(const struct cell_manager *src) -{ - struct cell_manager *pthis = cell_manager_new_without_map(src->sampling_mode, src->max_cell_num); - pthis->next_cell_id = src->next_cell_id; - - if (src->sampling_mode == SAMPLING_MODE_TOPK) { - pthis->topk_tag_id_map = heavy_keeper_copy(src->topk_tag_id_map); - - struct heavy_keeper_result *result = heavy_keeper_query(pthis->topk_tag_id_map); - for (size_t i = 0; i < result->n_key; i++) { - cell_manager_id_tag_array_add(pthis, result->cell_id[i], result->tags[i]); - } - heavy_keeper_result_free(result); - } else { - for (int i = 0; i <= src->max_cell_id; i++) { - const struct tag_hash_key *tag = src->id_tag_array[i]; - if (tag == NULL) { - continue; - } - struct tag_hash_key *tag_copy = tag_hash_key_copy(tag); - struct tag_id_map *tmp_node = (struct tag_id_map *)malloc(sizeof(struct tag_id_map)); - tmp_node->tag = tag_copy; - tmp_node->cell_id = i; - tmp_node->dying = false; - HASH_ADD_TAG(pthis->comprehensive_tag_id_map, tag, tmp_node); - - cell_manager_id_tag_array_add(pthis, i, tag_copy); - } - } - - return pthis; -} - -const struct tag_hash_key **cell_manager_dump(const struct cell_manager *pthis, int *array_len) -{ - *array_len = pthis->max_cell_id + 1; - - return (const struct tag_hash_key **)pthis->id_tag_array; -} - -int cell_manager_find(const struct cell_manager *pthis, const struct tag_hash_key *tag) -{ - if (pthis->sampling_mode == SAMPLING_MODE_COMPREHENSIVE) { - struct tag_id_map *node = NULL; - HASH_FIND_TAG(pthis->comprehensive_tag_id_map, tag, node); - if (node == NULL) { - return -1; - } else { - return node->dying ? -1 : node->cell_id; - } - } else { - int cell_id = -1; - heavy_keeper_query_one(pthis->topk_tag_id_map, tag, NULL, &cell_id); - return cell_id; - } -} - -void cell_manager_merge_topk(struct cell_manager *dest, const struct cell_manager *src, - int **cell_id_popped, int *n_cell_id_popped, int **cell_id_old, int **cell_id_added, int *n_cell_id_added) -{ - heavy_keeper_merge_recording_id_details(dest->topk_tag_id_map, src->topk_tag_id_map, - cell_id_popped, n_cell_id_popped, cell_id_old, cell_id_added, n_cell_id_added); - - memset(dest->id_tag_array, 0, sizeof(struct tag_hash_key *) * dest->id_tag_array_len); - struct heavy_keeper_result *result = heavy_keeper_query(dest->topk_tag_id_map); - for (size_t i = 0; i < result->n_key; i++) { - cell_manager_id_tag_array_add(dest, result->cell_id[i], result->tags[i]); - } - heavy_keeper_result_free(result); -} - -int cell_manager_get_cardinality(const struct cell_manager *pthis) -{ - if (pthis->sampling_mode == SAMPLING_MODE_COMPREHENSIVE) { - return pthis->current_cell_num; - } else { - return heavy_keeper_get_cardinality(pthis->topk_tag_id_map); - } -} - -void cube_manager_free(struct cube_manager *pthis) -{ - tag_id_map_free(pthis->head); - free(pthis); -} - -struct cube_manager *cube_manager_new() -{ - struct cube_manager *pthis = (struct cube_manager *)malloc(sizeof(struct cube_manager)); - pthis->head = NULL; - return pthis; -} - -void cube_manager_add(struct cube_manager *pthis, const struct tag_hash_key *tag, int id) -{ - struct tag_id_map *node = (struct tag_id_map *)malloc(sizeof(struct tag_id_map)); - node->tag = tag_hash_key_copy(tag); - node->cell_id = id; - node->dying = false; // this is not used in cube manager, just give it a value - HASH_ADD_TAG(pthis->head, tag, node); -} - -void cube_manager_delete(struct cube_manager *pthis, const struct tag_hash_key *tag) -{ - struct tag_id_map *node = NULL; - HASH_FIND_TAG(pthis->head, tag, node); - if (node != NULL) { - HASH_DEL(pthis->head, node); - tag_hash_key_free(node->tag); - free(node); - } -} - -int cube_manager_find(const struct cube_manager *pthis, const struct tag_hash_key *tag) -{ - struct tag_id_map *node = NULL; - HASH_FIND_TAG(pthis->head, tag, node); - if (node == NULL) { - return -1; - } else { - return node->cell_id; - } -} - -void cube_manager_calibrate(struct cube_manager *pthis, const struct cube_manager *master) -{ - struct tag_id_map *node = NULL; - struct tag_id_map *tmp = NULL; - HASH_ITER(hh, pthis->head, node, tmp) { - int cube_id = cube_manager_find(master, node->tag); - if (cube_id == -1) { - HASH_DEL(pthis->head, node); - tag_hash_key_free(node->tag); - free(node); - } else { - node->cell_id = cube_id; - } - } - - // exist in master but not in self - HASH_ITER(hh, master->head, node, tmp) { - int cube_id = cube_manager_find(pthis, node->tag); - if (cube_id == -1) { - cube_manager_add(pthis, node->tag, node->cell_id); - } - } -}
\ No newline at end of file diff --git a/src/tags/cell_manager.h b/src/tags/cell_manager.h deleted file mode 100644 index 479e444..0000000 --- a/src/tags/cell_manager.h +++ /dev/null @@ -1,41 +0,0 @@ - -#pragma once -#include "my_ut_hash.h" - -#ifdef __cplusplus -extern "C"{ -#endif - -struct cell_manager; - -void cell_manager_free(struct cell_manager *pthis); -struct cell_manager *cell_manager_new(enum sampling_mode sampling_mode, int max_cell_num); -struct cell_manager *cell_manager_copy(const struct cell_manager *src); -void cell_manager_reset(struct cell_manager *pthis); - -const struct tag_hash_key *cell_manager_get_tag_by_cell_id(const struct cell_manager *pthis, int cell_id); -long long cell_manager_get_count_by_tag(const struct cell_manager *pthis, const struct tag_hash_key *tag); -const struct tag_hash_key **cell_manager_dump(const struct cell_manager *pthis, int *array_len); -int cell_manager_find(const struct cell_manager *pthis, const struct tag_hash_key *tag); -int cell_manager_get_cardinality(const struct cell_manager *pthis); - -int cell_manager_add_cell(struct cell_manager *pthis, const struct tag_hash_key *tag); -int cell_manager_add_cell_topk(struct cell_manager *pthis, const struct tag_hash_key *tag, unsigned long long int count, int *popped_cell_id); -int cell_manager_delete_cell(struct cell_manager *pthis, const struct tag_hash_key *tag); - -void cell_manager_merge_topk(struct cell_manager *dest, const struct cell_manager *src, - int **cell_id_popped, int *n_cell_id_popped, int **cell_id_old, int **cell_id_added, int *n_cell_id_added); - -/* ------- a simpler cell_manager, which only support check if tag exist and add. ------- */ -struct cube_manager; - -struct cube_manager *cube_manager_new(); -void cube_manager_free(struct cube_manager *pthis); -void cube_manager_add(struct cube_manager *pthis, const struct tag_hash_key *tag, int id); -int cube_manager_find(const struct cube_manager *pthis, const struct tag_hash_key *tag); -void cube_manager_delete(struct cube_manager *pthis, const struct tag_hash_key *tag); -void cube_manager_calibrate(struct cube_manager *pthis, const struct cube_manager *master); - -#ifdef __cplusplus -} -#endif
\ No newline at end of file diff --git a/src/tags/heavy_keeper.c b/src/tags/heavy_keeper.c deleted file mode 100644 index b733dc6..0000000 --- a/src/tags/heavy_keeper.c +++ /dev/null @@ -1,563 +0,0 @@ -#include "heavy_keeper.h" - -#include <float.h> -#include <limits.h> -#include <math.h> -#include <stdbool.h> -#include <stddef.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <assert.h> - -#include "minheap/heap.h" -#include "mpack/mpack.h" -#include "xxhash/xxhash.h" -#include "cjson/cJSON.h" - -#include "tags/sorted_set.h" -#include "tags/my_ut_hash.h" - - -struct Bucket { - unsigned int finger_print; - unsigned long long count; // the actual count, sum of all uuid_hash_node->count -}; - -struct heavy_keeper { - int K; - struct sorted_set *top_K_heap; - struct heavy_keeper_options params; - struct Bucket *sketch; - - unsigned int rand_state; -}; - -#define FP_HASH_KEY 0 - -struct Bucket *new_sketch(struct heavy_keeper_options *params) { - size_t array_len = (size_t)params->array_num * (size_t)params->max_bucket_num; - - struct Bucket *ret = (struct Bucket *)calloc(array_len, sizeof(struct Bucket)); - - return ret; -} - -void params_set_to_default(struct heavy_keeper_options *p, int K) { - if (K > 1000) { // don't let the sketch too large when K gets insanely large - K = 1000; - } - double log_ret = log((double)K) / 2.3; // 2.3: log_e(10), log_ret = log_10(K) - - if (log_ret < 3) { - p->array_num = 3; - } else { - p->array_num = (int)(log_ret); - } - p->decay_exp_rate = 1.17; // by test, 1.17 is the best - p->max_bucket_num = (int)(log_ret * K * 2); - if (p->max_bucket_num < 150) { - p->max_bucket_num = 150; // determined through test, too small max_bucket_num will let accuracy decrease severely. - } else if (p->max_bucket_num > 600) { - p->max_bucket_num = p->max_bucket_num / 4 + 450; - } -} - -void params_set_to(struct heavy_keeper_options *p, const struct heavy_keeper_options *src) { - if (src->array_num != 0) { - p->array_num = src->array_num; - } - if (src->decay_exp_rate != 0) { - p->decay_exp_rate = src->decay_exp_rate; - } - if (src->max_bucket_num != 0) { - p->max_bucket_num = src->max_bucket_num; - } -} - -struct heavy_keeper *heavy_keeper_new(int max_query_num, const struct heavy_keeper_options *params) { - struct heavy_keeper *hk = (struct heavy_keeper *)malloc(sizeof(struct heavy_keeper)); - hk->K = max_query_num; - hk->rand_state = 0; - - hk->top_K_heap = sorted_set_new(max_query_num); - - params_set_to_default(&(hk->params), max_query_num); - - if (params != NULL) { - params_set_to(&(hk->params), params); - } - hk->sketch = new_sketch(&(hk->params)); - return hk; -} - -struct heavy_keeper *heavy_keeper_with_data_new(int max_query_num) { - struct heavy_keeper *hk = heavy_keeper_new(max_query_num, NULL); - return hk; -} - -void heavy_keeper_reset(struct heavy_keeper *hk) { - memset(hk->sketch, 0, (size_t)hk->params.array_num * (size_t)hk->params.max_bucket_num * sizeof(struct Bucket)); - sorted_set_reset(hk->top_K_heap); -} - -void heavy_keeper_result_free(struct heavy_keeper_result *stats_hd) { - free(stats_hd->count); - free(stats_hd->tags); - free(stats_hd->cell_id); - free(stats_hd); -} - -void heavy_keeper_free(struct heavy_keeper *hk) { - sorted_set_free(hk->top_K_heap); - free(hk->sketch); - free(hk); -} - -/*********************************add and query*************************************/ -/***********************************************************************************/ -const double DECAY_POW_TABLE[147] = { // 1.17 ^ exp, exp is in [0,146] - 1.0,0.8547008547009,0.7305135510264,0.6243705564328,0.5336500482332,0.456111152336,0.3898385917402,0.3331953775557,0.2847823739793,0.2434037384438,0.2080373832853,0.1778097293037,0.15197412761, - 0.1298924167607,0.1110191596245,0.0948881706192,0.0811010005293,0.0693170944695,0.0592453798884,0.0506370768277,0.0432795528442,0.036991070807,0.031616299835,0.0270224784915,0.0230961354628,0.0197402867204, - 0.0168720399319,0.0144205469504,0.0123252538037,0.0105344049605,0.0090037649235,0.0076955255756,0.0065773722868,0.0056216857153,0.0048048595857,0.0041067175946,0.0035100150381,0.0030000128531,0.0025641135497, - 0.0021915500424,0.0018731196944,0.0016009570038,0.0013683393194,0.0011695207859,0.0009995904153,0.0008543507823,0.0007302143438,0.0006241148238,0.0005334314733,0.0004559243362,0.0003896789198,0.0003330589058, - 0.0002846657315,0.000243304044,0.0002079521743,0.0001777369012,0.0001519118813,0.0001298392148,0.0001109736879,0.0000948493059,0.0000810677828,0.0000692887032,0.0000592211139,0.0000506163367,0.0000432618262, - 0.0000369759198,0.0000316033503,0.0000270114105,0.0000230866756,0.0000197322014,0.0000168651294,0.0000144146405,0.0000123202056,0.0000105300902,0.0000090000771,0.0000076923736,0.0000065746783,0.0000056193832, - 0.0000048028916,0.0000041050355,0.0000035085774,0.0000029987841,0.0000025630633,0.0000021906524,0.0000018723525,0.0000016003013,0.0000013677789,0.0000011690418,0.000000999181,0.0000008540009,0.0000007299153, - 0.0000006238592,0.000000533213,0.0000004557376,0.0000003895193,0.0000003329225,0.0000002845491,0.0000002432044,0.000000207867,0.0000001776641,0.0000001518497,0.000000129786,0.0000001109282,0.0000000948105, - 0.0000000810346,0.0000000692603,0.0000000591969,0.0000000505956,0.0000000432441,0.0000000369608,0.0000000315904,0.0000000270003,0.0000000230772,0.0000000197241,0.0000000168582,0.0000000144087,0.0000000123152, - 0.0000000105258,0.0000000089964,0.0000000076892,0.000000006572,0.0000000056171,0.0000000048009,0.0000000041034,0.0000000035071,0.0000000029976,0.000000002562,0.0000000021898,0.0000000018716,0.0000000015996, - 0.0000000013672,0.0000000011686,0.0000000009988,0.0000000008537,0.0000000007296,0.0000000006236,0.000000000533,0.0000000004556,0.0000000003894,0.0000000003328,0.0000000002844,0.0000000002431,0.0000000002078, - 0.0000000001776,0.0000000001518,0.0000000001297,0.0000000001109 -}; - -bool if_need_to_decay(struct heavy_keeper *hk, const struct Bucket *bucket, unsigned long long count) { - if (count == 0) { - return false; - } - if (bucket->count < count) { // the exp is 0, so the decay rate is 1 - return true; - } - - unsigned long long exp = bucket->count / count; - if (exp > 146) { // 1.17 ^ 146 < 1 / RAND_MAX < rand_r / RAND_MAX, so there is no chance to decay - return false; - } - - double r = (double)rand_r(&(hk->rand_state)) / (double)RAND_MAX; - - // double b = hk->params.decay_exp_rate; - // double decay_rate = pow(b, -exp); - // p->decay_exp_rate = 1.17 is fixed, search table to get result directly. - double decay_rate = DECAY_POW_TABLE[exp]; - - if (r < decay_rate) { - return true; - } - return false; -} - -struct Bucket *map_flow_id_hash_to_bucket(struct heavy_keeper *hk, int array_index, const struct tag_hash_key *tag) { - int w = hk->params.max_bucket_num; - int Hsh = tag_hash_key_cal_hash_val(tag, array_index + 1) % w; // +1: the let any row do not use FP as hashing key directly. - // Otherwise, when different key has the same FP(in 2^-64 probability), they will also in the same bucket in the row #0 and hash collision will happen more severely. - - return &(hk->sketch[array_index * w + Hsh]); -} - -unsigned long long count_add(unsigned long long a, unsigned long long b) { - if (a > UINT64_MAX - b) { - return UINT64_MAX; - } - return a + b; -} - -unsigned my_max(unsigned a, unsigned b) { - if (a > b) { - return a; - } - return b; -} - -static int heavy_keeper_add_by_recording_popped_data(struct heavy_keeper *heavy_keeper, const struct tag_hash_key *tag, unsigned long long int count, int cell_id, int *popped_id_out) -{ - if (count == 0) { - int cell_id_old = sorted_set_get_cell_id(heavy_keeper->top_K_heap, tag); - if (cell_id_old > 0) { - return 1; // the flow is in the sorted set. Use original cell id. - } - - assert(cell_id >= 0); - assert(tag != NULL); - long long tmp_v = tag->hashv; - assert(tmp_v >= 0); - - if (sorted_set_cardinality(heavy_keeper->top_K_heap) < heavy_keeper->K) { - sorted_set_insert_and_record_popped_back(heavy_keeper->top_K_heap, tag, 0, cell_id, popped_id_out); - return 0; // newly added - } - return -1; // not added - } - - int user_data_cube_id = 0; - struct sorted_set *summary = heavy_keeper->top_K_heap; - - unsigned nMin = sorted_set_get_min_count(summary); - if (nMin == INVALID_COUNT) { - nMin = 0; - } - unsigned long long int old_cnt = sorted_set_get_count(summary, tag); - bool not_in_sorted_set = (old_cnt == INVALID_COUNT); - unsigned long long maxv = 0; - - unsigned int FP = tag_hash_key_cal_hash_val(tag, FP_HASH_KEY); - for (int j = 0; j < heavy_keeper->params.array_num; j++) { - struct Bucket *bucket = map_flow_id_hash_to_bucket(heavy_keeper, j, tag); - - if (bucket->finger_print == FP) { - // If a flow is not in the min-heap, then the estimated flow size should be no larger than nmin. - // The flows whose counts are both larger than nmin and not in min-heap must have the same xxhash value, and its FP stored in bucket represents another tag, - // In this case, the sketch won't be updated. This flow is expected to be taken into account in another array, - // where its FP is different from the one it should collided with, so that element flows won't be missed. - if (bucket->count > nMin && not_in_sorted_set) { - continue; - } - bucket->count = count_add(bucket->count, count); - maxv = my_max(maxv, bucket->count); - } else { - if (!if_need_to_decay(heavy_keeper, bucket, count)) { - continue; - } - if (bucket->count < count) { - bucket->finger_print = FP; - bucket->count = count; - - maxv = my_max(maxv, count); - } else { - bucket->count -= count; - } - } - } - - - if (not_in_sorted_set) { - if ((maxv - nMin <= count && maxv != nMin) || sorted_set_cardinality(summary) != heavy_keeper->K) { - assert(cell_id >= 0); - assert(tag != NULL); - long long tmp_v = tag->hashv; - assert(tmp_v >= 0); - - int insert_ret = sorted_set_insert_and_record_popped_back(summary, tag, maxv, cell_id, &user_data_cube_id); - if (insert_ret == 2) { // insert success and popped one - *popped_id_out = user_data_cube_id; - return 0; - } else if (insert_ret == 0) { // insert failed - return -1; - } else if (insert_ret == 1) { // insert success without popping - return 0; - } else { - return 2; - } - } - return -1; // the flow is not added to the sorted set - } else { - if (maxv > old_cnt) { - sorted_set_incrby(summary, tag, maxv - old_cnt); - } - return 1; // no popped, but the tag definitely exists in the sorted set - } - - // should not reach here - return -1; -} - -int heavy_keeper_add(struct heavy_keeper *hk, const struct tag_hash_key *tag, unsigned long long int count, int cell_id, int *popped_id_out) -{ - *popped_id_out = -1; - heavy_keeper_add_by_recording_popped_data(hk, tag, count, cell_id, popped_id_out); - - int cell_id_tmp = sorted_set_get_cell_id(hk->top_K_heap, tag); - if (cell_id_tmp >= 0) { - if (cell_id_tmp == cell_id) { // insert success - return 0; - } else { - return 1; // insert success, but use the different cell_id - } - } else { - return -1; - } - - return -1; // should not reach here -} - -struct tag_hash_key *heavy_keeper_get_true_pst_tag(const struct heavy_keeper *hk, const struct tag_hash_key *tag) -{ - return sorted_set_get_true_pst_tag(hk->top_K_heap, tag); -} - -struct heavy_keeper_result *heavy_keeper_query(const struct heavy_keeper *hk) { - size_t query_entry_num = sorted_set_cardinality(hk->top_K_heap); - struct heavy_keeper_result *stats = (struct heavy_keeper_result *)malloc(sizeof(struct heavy_keeper_result)); - stats->count = (unsigned long long *)malloc(query_entry_num * sizeof(unsigned long long)); - stats->tags = (struct tag_hash_key **)malloc(query_entry_num * sizeof(struct tag_hash_key *)); - stats->cell_id = (int *)malloc(query_entry_num * sizeof(int)); - stats->n_key = query_entry_num; - - sorted_set_dump(hk->top_K_heap, stats->count, stats->tags, stats->cell_id); - - return stats; -} - -size_t heavy_keeper_get_cardinality(const struct heavy_keeper *hk) { - return sorted_set_cardinality(hk->top_K_heap); -} - -int heavy_keeper_query_one(const struct heavy_keeper *hk, const struct tag_hash_key *tag, unsigned long long int *count_out, int *cell_id_out) -{ - if (count_out != NULL) { - unsigned long long int count = sorted_set_get_count(hk->top_K_heap, tag); - if (count == INVALID_COUNT) { - *count_out = 0; - return -1; - } - *count_out = count; - } - - if (cell_id_out == NULL) { - return 0; - } - - int cell_id = sorted_set_get_cell_id(hk->top_K_heap, tag); - if (cell_id == -1) { - *cell_id_out = -1; - return -1; - } - *cell_id_out = cell_id; - - return 0; -} - -/*****************************************merge *****************************************/ -/****************************************************************************************/ -static void heavy_keeper_merge_sketch(struct heavy_keeper *dest, const struct heavy_keeper *src) { - int w = dest->params.max_bucket_num; - int d = dest->params.array_num; - //idx - for (int array_id = 0; array_id < d; array_id++) { - for (int bucket_id = 0; bucket_id < w; bucket_id++) { - struct Bucket *bucket_dest = &(dest->sketch[array_id * w + bucket_id]); - const struct Bucket *bucket_src = &(src->sketch[array_id * w + bucket_id]); - - if (bucket_dest->finger_print == bucket_src->finger_print) { - bucket_dest->count = count_add(bucket_dest->count, bucket_src->count); - } else { - if (bucket_dest->count < bucket_src->count) { // bucket_src stores the elephant flow. - bucket_dest->count = bucket_src->count; - bucket_dest->finger_print = bucket_src->finger_print; - } - } - } - } -} - -unsigned long long find_maxv_in_sketch(struct heavy_keeper *hk, const struct tag_hash_key *tag) { - struct Bucket *bucket; - unsigned fp = tag_hash_key_cal_hash_val(tag, FP_HASH_KEY); - - unsigned long long maxv = 0; - for (int array_id = 0; array_id < hk->params.array_num; array_id++) { - bucket = map_flow_id_hash_to_bucket(hk, array_id, tag); // hk->sketch is the merge of two. So just check one - - if (bucket->finger_print == fp) { - maxv = my_max(maxv, bucket->count); - } - } - - return maxv; -} - -int find_next_unused_cell_id(const int *sorted_cell_id_arr, size_t arr_len, int last_find_result, int *next_idx) -{ - int cur_idx = *next_idx; - - if (arr_len == 0) { - if (last_find_result < 0) { - return 0; - } else { - return last_find_result + 1; - } - } - - if (cur_idx >= arr_len) { - int max_id = sorted_cell_id_arr[arr_len - 1]; - if (last_find_result <= max_id) { // this first time reach the end of sorted_pst_cell_id_arr - return max_id + 1; - } else { - return last_find_result + 1; - } - } - - int expected_id = last_find_result + 1; - while (cur_idx < arr_len) { - int cur_id = sorted_cell_id_arr[cur_idx]; - - // check if expected_id id is used - if (cur_id == expected_id) { // used - expected_id++; - cur_idx++; - } else if (cur_id > expected_id) { // not used - *next_idx = cur_idx; - return expected_id; - } else { // still cannot guarantee expected_id is not used - cur_idx++; - } - } - - // all cell_id in sorted_pst_cell_id_arr is used - int max_id = sorted_cell_id_arr[arr_len - 1]; - *next_idx = arr_len; - return max_id + 1; -} - -int cmp_int(const void *a, const void *b) { - return *(int *)a - *(int *)b; -} - -void heavy_keeper_merge_sorted_set_recording_id_details(struct heavy_keeper *dest, const struct heavy_keeper *src, - int **cell_id_popped_from_dest_when_merge_src_out, int *cell_id_popped_from_dest_when_merge_src_len_out, - int **cell_id_in_src_before_merge_out, int **cell_id_in_dest_corresponding_out, int *cell_id_in_src_before_merge_len_out) -{ - struct sorted_set *new_rec = sorted_set_new(dest->K); // merging result - - struct sorted_set *ss_dest = dest->top_K_heap; // original ss in dest - const struct sorted_set *ss_src = src->top_K_heap; // the ss to be merged - int size_dest = sorted_set_cardinality(ss_dest); - int size_src = sorted_set_cardinality(ss_src); - int max_size = size_dest > size_src ? size_dest : size_src; - - unsigned long long *count_arr = (unsigned long long *)malloc(max_size * sizeof(unsigned long long)); - // struct tag_hash_key **tag_arr = (struct tag_hash_key **)malloc(max_size * sizeof(struct tag_hash_key *)); - struct tag_hash_key **tag_arr = (struct tag_hash_key **)calloc(max_size, sizeof(struct tag_hash_key *)); - int *cell_id_arr_dst = (int *)malloc(size_dest * sizeof(int)); - - sorted_set_dump(ss_dest, count_arr, tag_arr, cell_id_arr_dst); - int cell_id_popped_from_dest_when_merge_src[dest->K]; - int cell_id_popped_from_dest_when_merge_src_len = 0; - int cell_id_in_src_before_merge[dest->K]; - int cell_id_in_src_before_merge_len = 0; - int cell_id_in_dest_corresponding[dest->K]; // length == cell_id_in_src_before_merge_len - - /* ------------------------------ merge dest ------------------------------ */ - for (int i = 0; i < size_dest; i++) { - unsigned long long maxv = find_maxv_in_sketch(dest, tag_arr[i]); - assert(tag_arr[i] != NULL); - assert(cell_id_arr_dst[i] >= 0); - sorted_set_insert(new_rec, tag_arr[i], maxv, cell_id_arr_dst[i]); - } - - qsort(cell_id_arr_dst, size_dest, sizeof(int), cmp_int); // use in find_next_unused_cell_id to find the minimum unused cell_id - - /* ------------------------------ merge source ------------------------------ */ - // int *cell_id_arr_src = (int *)malloc(size_src * sizeof(int)); - int *cell_id_arr_src = (int*)calloc(size_src, sizeof(int)); - sorted_set_dump(ss_src, count_arr, tag_arr, cell_id_arr_src); - int last_find_id = -1; - int next_idx = 0; - - for (int i = 0; i < size_src; i++) { - const struct tag_hash_key *tag_src = tag_arr[i]; - assert(tag_src != NULL); - - int cell_id_dest = sorted_set_get_cell_id(ss_dest, tag_src); - int cell_id_src = cell_id_arr_src[i]; - - if (cell_id_dest != -1) { // the tag is in both dest and src, so has been processed in the previous loop. The reason why no need to sum up result is that merging sketch already gets its summed up count - cell_id_in_src_before_merge[cell_id_in_src_before_merge_len] = cell_id_src; - cell_id_in_dest_corresponding[cell_id_in_src_before_merge_len] = cell_id_dest; - cell_id_in_src_before_merge_len++; - - continue; - } - - unsigned long long maxv = find_maxv_in_sketch(dest, tag_src); // the dest heavy keeper has been merged, so the maxv is the maxv in the merged sketch, instead of the dest sketch - int popped_id = -1; - int now_idx = next_idx; - int find_id = find_next_unused_cell_id(cell_id_arr_dst, size_dest, last_find_id, &now_idx); - assert(find_id >= 0); - assert(now_idx >= 0); - int tmp_ret = sorted_set_insert_and_record_popped_back(new_rec, tag_src, maxv, find_id, &popped_id); - if (tmp_ret == 2) { // insert success and popped one. - cell_id_popped_from_dest_when_merge_src[cell_id_popped_from_dest_when_merge_src_len++] = popped_id; - } - if (tmp_ret != 0) { // insert success - last_find_id = find_id; - next_idx = now_idx; - - cell_id_in_src_before_merge[cell_id_in_src_before_merge_len] = cell_id_src; - cell_id_in_dest_corresponding[cell_id_in_src_before_merge_len] = last_find_id; - cell_id_in_src_before_merge_len++; - } - } - - free(count_arr); - free(tag_arr); - free(cell_id_arr_dst); - free(cell_id_arr_src); - sorted_set_free(ss_dest); - - dest->top_K_heap = new_rec; - - if (cell_id_popped_from_dest_when_merge_src_len == 0) { - *cell_id_popped_from_dest_when_merge_src_out = NULL; - *cell_id_popped_from_dest_when_merge_src_len_out = 0; - } else { - *cell_id_popped_from_dest_when_merge_src_out = (int *)malloc(cell_id_popped_from_dest_when_merge_src_len * sizeof(int)); - memcpy(*cell_id_popped_from_dest_when_merge_src_out, cell_id_popped_from_dest_when_merge_src, cell_id_popped_from_dest_when_merge_src_len * sizeof(int)); - *cell_id_popped_from_dest_when_merge_src_len_out = cell_id_popped_from_dest_when_merge_src_len; - } - if (cell_id_in_src_before_merge_len == 0) { - *cell_id_in_src_before_merge_out = NULL; - *cell_id_in_src_before_merge_len_out = 0; - *cell_id_in_dest_corresponding_out = NULL; - } else { - *cell_id_in_src_before_merge_out = (int *)malloc(cell_id_in_src_before_merge_len * sizeof(int)); - memcpy(*cell_id_in_src_before_merge_out, cell_id_in_src_before_merge, cell_id_in_src_before_merge_len * sizeof(int)); - *cell_id_in_src_before_merge_len_out = cell_id_in_src_before_merge_len; - *cell_id_in_dest_corresponding_out = (int *)malloc(cell_id_in_src_before_merge_len * sizeof(int)); - memcpy(*cell_id_in_dest_corresponding_out, cell_id_in_dest_corresponding, cell_id_in_src_before_merge_len * sizeof(int)); - } -} - -int heavy_keeper_merge_recording_id_details(struct heavy_keeper *dest, const struct heavy_keeper *src, - int **cell_id_popped_from_dest_when_merge_src_out, int *cell_id_popped_from_dest_when_merge_src_len_out, - int **cell_id_in_src_before_merge_out, int **cell_id_in_dest_corresponding_out, int *cell_id_in_src_before_merge_len_out) -{ - if (dest->params.array_num != src->params.array_num || - dest->params.max_bucket_num != src->params.max_bucket_num || - dest->K != src->K) { - return -1; - } - - heavy_keeper_merge_sketch(dest, src); - heavy_keeper_merge_sorted_set_recording_id_details(dest, src, - cell_id_popped_from_dest_when_merge_src_out, cell_id_popped_from_dest_when_merge_src_len_out, - cell_id_in_src_before_merge_out, cell_id_in_dest_corresponding_out, cell_id_in_src_before_merge_len_out); - return 0; -} - -struct heavy_keeper *heavy_keeper_copy(const struct heavy_keeper *src) -{ - struct heavy_keeper *hk = (struct heavy_keeper *)malloc(sizeof(struct heavy_keeper)); - hk->params = src->params; - hk->K = src->K; - hk->sketch = (struct Bucket *)malloc(hk->params.array_num * hk->params.max_bucket_num * sizeof(struct Bucket)); - memcpy(hk->sketch, src->sketch, hk->params.array_num * hk->params.max_bucket_num * sizeof(struct Bucket)); - - hk->top_K_heap = sorted_set_copy(src->top_K_heap); - return hk; -} - -int heavy_keeper_get_config_K(const struct heavy_keeper *hk) -{ - return hk->K; -} diff --git a/src/tags/heavy_keeper.h b/src/tags/heavy_keeper.h deleted file mode 100644 index 688911b..0000000 --- a/src/tags/heavy_keeper.h +++ /dev/null @@ -1,110 +0,0 @@ -#pragma once - -#include <stddef.h> -#include <uuid/uuid.h> - -/* --------------------------------------------------------------------------------------- -TOP K using heavykeeper algorithm. -Author: Chen Zizhan -Time: 2023/4/26 ----------------------------------------------------------------------------------------------*/ - -#ifdef __cplusplus -extern "C"{ -#endif - - -// Query result -struct heavy_keeper_result { - size_t n_key; // number of result - struct tag_hash_key **tags; - unsigned long long *count; // count[i] is the metric count of the corresponding flow id - - int *cell_id; -}; - -// Parameters used in algorithm -struct heavy_keeper_options{ - int array_num; // the size of the array. Default value: 4 - // Set it by how accurate you want. Value of 4 guarantees an accuracy more than 90% and around 97% in all tests. - // Not too big because it have an impact on both time and memory efficiency. - int max_bucket_num; // M2, the maximum number of buckets every array keeps. Default value: k*log(k) and no less than 100. - // Basically, as long as big enough, it won't affect the accuracy significantly. - double decay_exp_rate; // b, bigger variance of flow size is(elephant flows take more ratio), smaller it should be. - // Must bigger than 1. Better not bigger than 1.3, otherwise some elephant flow will be missed. -}; - -struct heavy_keeper; - -struct heavy_keeper *heavy_keeper_with_data_new(int max_query_num); -/** - * @brief free a heavy keeper. - * @param hk the pointer to the heavy keeper. - */ -void heavy_keeper_free(struct heavy_keeper *hk); - -/* - * @brief clear a heavy keeper. It will clear all the data in the heavy keeper. - * @param hk the pointer to the heavy keeper. -*/ -void heavy_keeper_reset(struct heavy_keeper *hk); - -// tag: the pointer to the tag, user should alloc it. Its memory will be freed by heavy keeper, so don't free it after calling heavy_keeper_add. -// return -1 when the key is not inserted to the heavy keeper(not fail). popped_id_out is -1 if not popped. any other value >= 0 means the id of the popped member. -// the cube id will give the key if the key is new, in which case return 0, if the key is already in the heavy keeper, return 1. -int heavy_keeper_add(struct heavy_keeper *hk, const struct tag_hash_key *tag, unsigned long long int count, int cube_id, int *popped_id_out); - -/** - * @brief Query the top-K flows.The flow id with bigger count ranks at smaller index. - * @param hk the pointer to the heavy keeper. - * @return a pointer to the heavy keeper query result. User should free it after use by calling heavy_keeper_result_free. - */ -struct heavy_keeper_result *heavy_keeper_query(const struct heavy_keeper *hk); - -int heavy_keeper_query_one(const struct heavy_keeper *hk, const struct tag_hash_key *tag, unsigned long long int *count_out, int *cube_id_out); - -size_t heavy_keeper_get_cardinality(const struct heavy_keeper *hk); -/** - * @brief free a heavy keeper query result. - * @param stats_hd the pointer to the heavy keeper query result. - */ -void heavy_keeper_result_free(struct heavy_keeper_result *stats_hd); - -int heavy_keeper_merge_recording_id_details(struct heavy_keeper *dest, const struct heavy_keeper *src, - int **cell_id_popped_from_dest_when_merge_src_out, int *cell_id_popped_from_dest_when_merge_src_len_out, - int **cell_id_in_src_before_merge_out, int **cell_id_in_dest_corresponding_out, int *cell_id_in_src_before_merge_len_out); - -/* - Equal to heavy_keeper_merge, with empty dest, but much faster. -*/ -struct heavy_keeper *heavy_keeper_copy(const struct heavy_keeper *src); - -/** - * @brief Serialize a heavy keeper. - * @param hk the pointer to the heavy keeper. - * @param blob output of the serialized data. Must be freed by caller. *blob can be NULL, the function will allocate memory for it. - * @param blob_sz Output, the size of the serialized data. -*/ -void heavy_keeper_serialization(const struct heavy_keeper *hk, char **blob, size_t *blob_sz); - -/** - * @brief Deserialize a heavy keeper. - * @param blob the serialized data. From heavy_keeper_serialization. - * @param size the size of the serialized data. - * @returns a pointer to the heavy keeper. -*/ -struct heavy_keeper *heavy_keeper_deserialization(const char *blob, size_t size); - - -/** - * @brief Get the "K" parameter of the heavy keeper. -*/ -int heavy_keeper_get_config_K(const struct heavy_keeper *hk); - -void heavy_keeper_get_config_metric_header(struct heavy_keeper *hk, char **metric_names, size_t *n_metric, size_t *primary_metric_index); - -struct tag_hash_key *heavy_keeper_get_true_pst_tag(const struct heavy_keeper *hk, const struct tag_hash_key *tag); - -#ifdef __cplusplus -} -#endif
\ No newline at end of file diff --git a/src/tags/my_ut_hash.c b/src/tags/my_ut_hash.c deleted file mode 100644 index 8123456..0000000 --- a/src/tags/my_ut_hash.c +++ /dev/null @@ -1,467 +0,0 @@ -#include <stdbool.h> -#include <stddef.h> -#include <stdio.h> -#include <stdlib.h> -#include <assert.h> -#include <string.h> -#define XXH_INLINE_ALL -#include "xxhash/xxhash.h" -#include "my_ut_hash_inner.h" -#include "my_ut_hash.h" -#include "serializer.h" - - -unsigned int tag_hash_key_cal_hash_value(const struct tag_hash_key *my_hash_key) -{ - return my_hash_key->hashv; -} - -// used by HASH_FIND -int tag_hash_key_cmp(const struct tag_hash_key *a, const struct tag_hash_key *b) -{ - if (a->n_my_tag == 0 && b->n_my_tag == 0) { - return 0; - } - - if (a->n_my_tag != b->n_my_tag) { - return 1; - } - if (a->hashv128.high64 != b->hashv128.high64 || a->hashv128.low64 != b->hashv128.low64) { - return 1; - } - - size_t tmp_tag_n = a->n_my_tag; - const struct fieldstat_tag *tag_a = a->tags; - const struct fieldstat_tag *tag_b = b->tags; - for (size_t i = 0; i < tmp_tag_n; i++) { - if (tag_a->type == TAG_CSTRING) { - if (strcmp(tag_a->value_str, tag_b->value_str) != 0) { - return 1; - } - } else { - if (tag_a->value_longlong != tag_b->value_longlong) { // sizeof(long long) == sizeof(double) - return 1; - } - } - - // its hardly possible that the keys are different while the value is the same. What's more, the tags are in the same 128 hash. So it's not necessary to compare the key. - // if (strcmp(tag_a->key, tag_b->key) != 0) { - // return 1; - // } - - tag_a++; - tag_b++; - } - return 0; -} - -/* -------------------------------------------------------------------------- */ -/* fieldtag */ -/* -------------------------------------------------------------------------- */ - -void fieldtag_copy(struct fieldstat_tag *dest, const struct fieldstat_tag *src) -{ - dest->key = strdup(src->key); - dest->type = src->type; - switch(src->type) - { - case TAG_INTEGER: - dest->value_longlong = src->value_longlong; - break; - case TAG_DOUBLE: - dest->value_double = src->value_double; - break; - case TAG_CSTRING: - dest->value_str = strdup(src->value_str); - break; - default: - assert(0); - break; - } -} - -void fieldtag_list_free(struct fieldstat_tag *tags, size_t n_tags) -{ - if (tags == NULL) { - return; - } - for (int i = 0; i < (int)n_tags; i++) { - struct fieldstat_tag *tag = &tags[i]; - free((char *)tag->key); - if (tag->type == TAG_CSTRING) { - free((char *)tag->value_str); - } - } - free(tags); -} - -int my_cmp_tag(const void *a, const void *b) -{ - const struct fieldstat_tag *aa = (struct fieldstat_tag *)a; - const struct fieldstat_tag *bb = (struct fieldstat_tag *)b; - int key_cmp = strcmp(aa->key, bb->key); - if (key_cmp != 0) { - return key_cmp; - } - if (aa->type != bb->type) { - return aa->type - bb->type; - } - - switch (aa->type) { - case TAG_INTEGER: - return aa->value_longlong - bb->value_longlong; - case TAG_DOUBLE: - return aa->value_double - bb->value_double; - case TAG_CSTRING: - return strcmp(aa->value_str, bb->value_str); - default: - return 0; - } -} - -unsigned int cal_tag_hash_xxhash(const struct fieldstat_tag *tag, size_t n_tag, unsigned int seed) -{ - XXH32_state_t state; - XXH32_reset(&state, seed); - - for (int i = 0; i < n_tag; i++) - { - XXH32_update(&state, (const xxh_u8 *)tag[i].key, strlen(tag[i].key)); - - if (tag[i].type != TAG_CSTRING) { // sizeof(long long) == sizeof(double) - XXH32_update(&state, (const xxh_u8 *)&tag[i].value_longlong, sizeof(long long)); - } else { - XXH32_update(&state, (const xxh_u8 *)tag[i].value_str, strlen(tag[i].value_str)); - } - } - - return XXH32_digest(&state); -} - -XXH128_hash_t cal_tag_hash_xxhash128_obsolete(const struct fieldstat_tag *tag, size_t n_tag, unsigned int seed) -{ - XXH3_state_t state = {0}; - XXH3_128bits_reset_withSeed(&state, seed); - - for (int i = 0; i < n_tag; i++) - { - XXH3_128bits_update(&state, tag[i].key, strlen(tag[i].key)); - - if (tag[i].type != TAG_CSTRING) { // sizeof(long long) == sizeof(double) - XXH3_128bits_update(&state, &tag[i].value_longlong, sizeof(long long)); - } else { - XXH3_128bits_update(&state, tag[i].value_str, strlen(tag[i].value_str)); - } - } - - XXH128_hash_t ret = XXH3_128bits_digest(&state); - - return ret; -} - -#define BIG_CONSTANT(x) (x##LLU) -#define getblock(p, i) (p[i]) -#ifdef __GNUC__ -#define FORCE_INLINE __attribute__((always_inline)) inline -#else -#define FORCE_INLINE inline -#endif -static FORCE_INLINE uint64_t rotl64 ( uint64_t x, int8_t r ) -{ - return (x << r) | (x >> (64 - r)); -} -#define ROTL64(x,y) rotl64(x,y) -#define C1 0x87c37b91114253d5LLU -#define C2 0x4cf5ad432745937fLLU - -static FORCE_INLINE uint64_t fmix64 ( uint64_t k ) -{ - k ^= k >> 33; - k *= BIG_CONSTANT(0xff51afd7ed558ccd); - k ^= k >> 33; - k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53); - k ^= k >> 33; - - return k; -} - -static FORCE_INLINE void murmurhash_string(uint64_t *h1_io, uint64_t *h2_io, const char *key, const int len) { - uint64_t h1 = *h1_io; - uint64_t h2 = *h2_io; - const uint8_t * data = (const uint8_t*)key; - const int nblocks = len / 16; - int i; - - // body - const uint64_t * blocks = (const uint64_t *)(data); - - for(i = 0; i < nblocks; i++) - { - uint64_t k1 = getblock(blocks,i*2+0); - uint64_t k2 = getblock(blocks,i*2+1); - - k1 *= C1; k1 = ROTL64(k1,31); k1 *= C2; h1 ^= k1; - - h1 = ROTL64(h1,27); h1 += h2; h1 = h1*5+0x52dce729; - - k2 *= C2; k2 = ROTL64(k2,33); k2 *= C1; h2 ^= k2; - - h2 = ROTL64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5; - } - - //---------- - // tail - - const uint8_t * tail = (const uint8_t*)(data + nblocks*16); - - uint64_t k1 = 0; - uint64_t k2 = 0; - - switch(len & 15) - { - case 15: k2 ^= (uint64_t)(tail[14]) << 48; - case 14: k2 ^= (uint64_t)(tail[13]) << 40; - case 13: k2 ^= (uint64_t)(tail[12]) << 32; - case 12: k2 ^= (uint64_t)(tail[11]) << 24; - case 11: k2 ^= (uint64_t)(tail[10]) << 16; - case 10: k2 ^= (uint64_t)(tail[ 9]) << 8; - case 9: k2 ^= (uint64_t)(tail[ 8]) << 0; - k2 *= C2; k2 = ROTL64(k2,33); k2 *= C1; h2 ^= k2; - - case 8: k1 ^= (uint64_t)(tail[ 7]) << 56; - case 7: k1 ^= (uint64_t)(tail[ 6]) << 48; - case 6: k1 ^= (uint64_t)(tail[ 5]) << 40; - case 5: k1 ^= (uint64_t)(tail[ 4]) << 32; - case 4: k1 ^= (uint64_t)(tail[ 3]) << 24; - case 3: k1 ^= (uint64_t)(tail[ 2]) << 16; - case 2: k1 ^= (uint64_t)(tail[ 1]) << 8; - case 1: k1 ^= (uint64_t)(tail[ 0]) << 0; - k1 *= C1; k1 = ROTL64(k1,31); k1 *= C2; h1 ^= k1; - }; - - *h1_io = h1; - *h2_io = h2; -} - -static FORCE_INLINE void murmurhash_int64(uint64_t *h2_io, uint64_t h1, long long value) { - // uint64_t h1 = *h1_io; - uint64_t h2 = *h2_io; - uint64_t k2 = value; - - k2 *= C2; k2 = ROTL64(k2,33); k2 *= C1; h2 ^= k2; - h2 = ROTL64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5; - - // *h1_io = h1; - *h2_io = h2; -} - -XXH128_hash_t my_murmurHash_128_tag(const struct fieldstat_tag *tag, size_t n_tag, unsigned int seed) -{ - uint64_t h1 = seed; - uint64_t h2 = seed; - const struct fieldstat_tag *tmp_tag; - - for (int i = 0; i < n_tag; i++) { - tmp_tag = tag + i; - - murmurhash_string(&h1, &h2, tmp_tag->key, strlen(tmp_tag->key)); - if (tmp_tag->type != TAG_CSTRING) { - murmurhash_int64(&h2, h1, tmp_tag->value_longlong); // sizeof(long long) == sizeof(double) - } else { - murmurhash_string(&h1, &h2, tmp_tag->value_str, strlen(tmp_tag->value_str)); - } - } - - // finalization - h1 += h2; - h2 += h1; - - h1 = fmix64(h1); - h2 = fmix64(h2); - - h1 += h2; - h2 += h1; - - XXH128_hash_t ret; - ret.low64 = h1; - ret.high64 = h2; - return ret; -} - -XXH128_hash_t cal_tag_hash_xxhash128(const struct fieldstat_tag *tag, size_t n_tag, unsigned int seed) { - return my_murmurHash_128_tag(tag, n_tag, seed); -} - -void tag_hash_key_init_with_fieldstat_tag(struct tag_hash_key *tag_key, const struct fieldstat_tag *src, size_t n_src, bool deep_copy) -{ - tag_key->n_my_tag = n_src; - tag_key->is_deep_copy = deep_copy; - if (n_src == 0) { - tag_key->tags = NULL; - tag_key->hashv = 0; - return; - } - - tag_key->hashv128 = cal_tag_hash_xxhash128(src, n_src, 0); - tag_key->hashv = tag_key->hashv128.high64; - - if (deep_copy) { - tag_key->tags = (struct fieldstat_tag *)malloc(sizeof(struct fieldstat_tag) * n_src); - for (int i = 0; i < (int)n_src; i++) { - fieldtag_copy(&tag_key->tags[i], &src[i]); - } - } else { - tag_key->tags = (struct fieldstat_tag *)src; - } -} - -void tag_hash_key_convert_to_fieldstat_tag(const struct tag_hash_key *tag_key, struct fieldstat_tag **out, size_t *n_out) -{ - size_t ret_n_tag = tag_key->n_my_tag; - *n_out = ret_n_tag; - if (ret_n_tag == 0) { - *out = NULL; - return; - } - - struct fieldstat_tag *ret_fieldstat_tag = (struct fieldstat_tag *)malloc(sizeof(struct fieldstat_tag) * ret_n_tag); - *out = ret_fieldstat_tag; - - for (int i = 0; i < ret_n_tag; i++) { - fieldtag_copy(&ret_fieldstat_tag[i], &tag_key->tags[i]); - } -} - -struct tag_hash_key *tag_hash_key_copy(const struct tag_hash_key *src) -{ - struct tag_hash_key *dst = (struct tag_hash_key *)malloc(sizeof(struct tag_hash_key)); - dst->n_my_tag = src->n_my_tag; - dst->hashv = src->hashv; - dst->is_deep_copy = true; - dst->hashv128 = src->hashv128; - - if (dst->n_my_tag == 0) { - dst->tags = NULL; - } else { - dst->tags = (struct fieldstat_tag *)malloc(sizeof(struct fieldstat_tag) * dst->n_my_tag); - for (int i = 0; i < dst->n_my_tag; i++) { - fieldtag_copy(&dst->tags[i], &src->tags[i]); - } - } - - return dst; -} - -void tag_hash_key_free(struct tag_hash_key *tag_key) -{ - if (tag_key->is_deep_copy) { - fieldtag_list_free(tag_key->tags, tag_key->n_my_tag); - } - free(tag_key); -} - -unsigned tag_hash_key_cal_hash_val(const struct tag_hash_key *tag, unsigned seed) { - if (seed == 0) { // use default seed - return tag->hashv; - } - - // https://www.eecs.harvard.edu/~michaelm/postscripts/tr-02-05.pdf - // A technique from the hashing literature is to use two hash functions h1(x) and h2(x) to simulate additional hash functions of the form gi(x) = h1(x) + ih2(x) - // Assuming that the 128-bit xxhash function is perfect, we can view it as a combination of two 64-bit hash functions. - unsigned h1 = (unsigned)tag->hashv128.high64; - unsigned h2 = (unsigned)tag->hashv128.low64; - unsigned ret = h1 + seed * h2; - - return ret; -} - -void build_dynamic_cell_key(const struct fieldstat_tag tags[], size_t n_tags, char **out_key, size_t *out_key_size) -{ - if (n_tags == 0) { - // use a default dummy key - *out_key = strdup("no tags"); - *out_key_size = strlen(*out_key); - return; - } - int i = 0; - int used_len = 0; - struct fieldstat_tag *tag = NULL; - - const size_t init_key_size = 1024; - size_t one_time_key_size = init_key_size; - char *one_time_key = (char *)malloc(init_key_size); - size_t remain_key_size = init_key_size; - size_t total_key_size = init_key_size; - char *dynamic_mem = (char *)malloc(remain_key_size); - size_t key_len; - size_t val_len; - - for(i = 0; i < (int)n_tags; i++) - { - tag = (struct fieldstat_tag *)&tags[i]; - switch(tag->type) - { - case 0: - key_len = strlen(tag->key); - val_len = sizeof(long long); - used_len = key_len + val_len; - while (used_len >= one_time_key_size) { - one_time_key_size += init_key_size; - one_time_key = (char *)realloc(one_time_key, one_time_key_size); - } - memcpy(one_time_key, tag->key, key_len); - memcpy(one_time_key + key_len, &tag->value_longlong, val_len); - break; - case 1: - key_len = strlen(tag->key); - val_len = sizeof(double); - used_len = key_len + val_len; - while (used_len >= one_time_key_size) { - one_time_key_size += init_key_size; - one_time_key = (char *)realloc(one_time_key, one_time_key_size); - } - memcpy(one_time_key, tag->key, key_len); - memcpy(one_time_key + key_len, &tag->value_double, val_len); - break; - case 2: - key_len = strlen(tag->key); - val_len = strlen(tag->value_str); - used_len = key_len + val_len; - while (used_len >= one_time_key_size) { - one_time_key_size += init_key_size; - one_time_key = (char *)realloc(one_time_key, one_time_key_size); - } - memcpy(one_time_key, tag->key, key_len); - memcpy(one_time_key + key_len, tag->value_str, val_len); - break; - default: - assert(0); - break; - } - - while (used_len >= remain_key_size) { - total_key_size += init_key_size; - dynamic_mem = (char *)realloc(dynamic_mem, total_key_size); - remain_key_size += init_key_size; - } - - memcpy(dynamic_mem + total_key_size - remain_key_size, one_time_key, used_len); - remain_key_size -= used_len; - } - - *out_key = (char *)malloc(total_key_size - remain_key_size + 1); - memcpy(*out_key, dynamic_mem, total_key_size - remain_key_size); - (*out_key)[total_key_size - remain_key_size] = '\0'; - *out_key_size = total_key_size - remain_key_size; // not include '\0' - - free(dynamic_mem); - free(one_time_key); -} - -char *tag_hash_key_get_compound_key(const struct tag_hash_key *tag) -{ - char *key; - size_t key_size; - build_dynamic_cell_key(tag->tags, tag->n_my_tag, &key, &key_size); - return key; -}
\ No newline at end of file diff --git a/src/tags/my_ut_hash.h b/src/tags/my_ut_hash.h deleted file mode 100644 index 1066569..0000000 --- a/src/tags/my_ut_hash.h +++ /dev/null @@ -1,32 +0,0 @@ - -#pragma once - -#ifdef __cplusplus -extern "C" -{ -#endif - -#include "fieldstat.h" -#include "xxhash/xxhash.h" - -struct tag_hash_key { - struct fieldstat_tag *tags; // not used for hash, just to recover the original tags. - size_t n_my_tag; - unsigned int hashv; - XXH128_hash_t hashv128; - bool is_deep_copy; -}; - -void tag_hash_key_init_with_fieldstat_tag(struct tag_hash_key *tag_key, const struct fieldstat_tag *src, size_t n_src, bool deep_copy); -void tag_hash_key_convert_to_fieldstat_tag(const struct tag_hash_key *tag_key, struct fieldstat_tag **out, size_t *n_out); -struct tag_hash_key *tag_hash_key_copy(const struct tag_hash_key *src); // Always deep copy -void tag_hash_key_free(struct tag_hash_key *tag_key); -unsigned tag_hash_key_cal_hash_val(const struct tag_hash_key *tag, unsigned seed); - -void fieldtag_list_free(struct fieldstat_tag *tags, size_t n_tags); -char *tag_hash_key_get_compound_key(const struct tag_hash_key *tag); -int tag_hash_key_cmp(const struct tag_hash_key *a, const struct tag_hash_key *b); - -#ifdef __cplusplus -} -#endif
\ No newline at end of file diff --git a/src/tags/my_ut_hash_inner.h b/src/tags/my_ut_hash_inner.h deleted file mode 100644 index 628b471..0000000 --- a/src/tags/my_ut_hash_inner.h +++ /dev/null @@ -1,38 +0,0 @@ - -#pragma once - -#ifdef __cplusplus -extern "C" -{ -#endif - -#include <stdbool.h> -#include <stddef.h> - -struct tag_hash_key; - -unsigned int tag_hash_key_cal_hash_value(const struct tag_hash_key *my_hash_key); -// return 1 if not equal, 0 if equal -int tag_hash_key_cmp(const struct tag_hash_key *a, const struct tag_hash_key *b); - -// cover the original HASH_FUNCTION -#define HASH_FUNCTION(keyptr, keylen, hashv) \ - do { \ - hashv = tag_hash_key_cal_hash_value((struct tag_hash_key *)keyptr); \ - } while (0) - -#define HASH_KEYCMP(a, b, n) (tag_hash_key_cmp((const struct tag_hash_key *)a, (const struct tag_hash_key *)b)) - -#include "uthash.h" - -#define HASH_ADD_TAG(head,fieldname,add) \ - HASH_ADD_KEYPTR(hh, head, (add)->fieldname, 1, add) - -#define HASH_FIND_TAG(head,keyptr,out) \ - HASH_FIND(hh, head, keyptr, 1, out) - - - -#ifdef __cplusplus -} -#endif
\ No newline at end of file diff --git a/src/tags/sorted_set.c b/src/tags/sorted_set.c deleted file mode 100644 index ae315d5..0000000 --- a/src/tags/sorted_set.c +++ /dev/null @@ -1,464 +0,0 @@ -
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-#include <assert.h>
-
-#include "my_ut_hash_inner.h"
-#include "xxhash/xxhash.h"
-#include "minheap/heap.h"
-
-#include "sorted_set.h"
-#include "my_ut_hash.h"
-
-
-struct entry_data {
- struct tag_hash_key *key;
-
- int cell_id;
-};
-
-struct sorted_set {
- heap *heap;
- struct hash_node *hash_tbl;
-
- int n_living_entry;
-};
-
-struct hash_node {
- struct tag_hash_key *key; // value is equal to entry_data::key(point to the same memory)
- heap_entry *val;
- UT_hash_handle hh;
-};
-
-struct entry_data *entry_data_construct(const struct tag_hash_key *tag, int cell_id)
-{
- assert(cell_id >= 0);
- assert(tag != NULL);
- long long tmp_v = tag->hashv;
- assert(tmp_v >= 0);
-
- struct entry_data *entry_data = (struct entry_data *)malloc(sizeof(struct entry_data));
-
- entry_data->key = tag_hash_key_copy(tag);
- entry_data->cell_id = cell_id;
- return entry_data;
-}
-
-void entry_data_destroy(struct entry_data *entry_data)
-{
- if (entry_data == NULL) {
- return;
- }
- tag_hash_key_free(entry_data->key);
- free(entry_data);
-}
-
-unsigned long long sorted_set_entry_get_count(const heap_entry *entry)
-{
- unsigned long long count = *(unsigned long long *)entry->key;
- if (count == 0) {
- return 0;
- }
- return count - 1; // sorted set will let the count start from 1, 0 for dying entry.
-}
-
-struct entry_data *sorted_set_entry_get_data(const heap_entry *entry)
-{
- assert(entry != NULL);
- assert(entry->value != NULL);
-
- const struct entry_data * tmp_val = (struct entry_data *)entry->value;
- assert(tmp_val->key != NULL);
- long long tmp_v = tmp_val->key->hashv;
- assert(tmp_v >= 0);
-
- assert(tmp_val->cell_id >= 0);
- return entry->value;
-}
-
-/*
- dying: To reduce the time of HASH_ADD, and to support merely recording a key without any value, give a dying status.
-*/
-bool sorted_set_entry_dying(const heap_entry *entry)
-{
- unsigned long long count = *(unsigned long long *)entry->key;
- return count == 0;
-}
-
-// cppcheck-suppress [constParameterCallback, unmatchedSuppression]
-static bool cmp_int(void *aa, void *bb)
-{
- heap_entry *a = (heap_entry *)aa;
- heap_entry *b = (heap_entry *)bb;
- unsigned long long count_a = *(unsigned long long *)a->key;
- unsigned long long count_b = *(unsigned long long *)b->key;
-
- if (count_a < count_b) {
- return true;
- }
- return false;
-}
-
-static int my_entry_get_index(void *a)
-{
- return ((heap_entry *)a)->index;
-}
-
-void my_entry_set_index(int index, void *a)
-{
- ((heap_entry *)a)->index = index;
-}
-
-struct sorted_set *sorted_set_new(int set_size)
-{
- struct sorted_set *ret = (struct sorted_set *)malloc(sizeof(struct sorted_set));
- ret->hash_tbl = NULL;
- ret->n_living_entry = 0;
- heap *h = (heap *)malloc(sizeof(heap));
- init_heap(h, set_size, cmp_int, my_entry_get_index, my_entry_set_index);
-
- ret->heap = h;
- return ret;
-}
-
-void sorted_set_free(struct sorted_set *ss)
-{
- struct hash_node *tbl = ss->hash_tbl;
- struct hash_node *h_node, *tmp;
- HASH_ITER(hh, tbl, h_node, tmp) {
- HASH_DEL(tbl, h_node);
- free(h_node);
- }
-
- heap *heap = ss->heap;
- heap_entry *node;
- for (int i = 0; i < heap->cur_size; i++) {
- node = (heap_entry *)heap->nodes[i];
- entry_data_destroy(node->value);
- free(node->key);
- free(node);
- }
- free(heap->nodes);
- free(heap);
-
- free(ss);
-}
-
-heap_entry *sorted_set_find_entry(const struct sorted_set *ss, const struct tag_hash_key *tag)
-{
- struct hash_node *hash_tbl = ss->hash_tbl;
- struct hash_node *s = NULL;
- HASH_FIND_TAG(hash_tbl, tag, s);
-
- if (s == NULL) {
- return NULL;
- }
- return (heap_entry *)s->val;
-}
-
-void sorted_set_entry_set_key(struct sorted_set *ss, heap_entry *entry, unsigned long long count)
-{
- *(unsigned long long *)(entry->key) = count;
-
- adjust_heap_node(ss->heap, entry);
-}
-
-unsigned long long sorted_set_pop_and_record_popped_back(struct sorted_set *ss, int *popped_cell_id)
-{
- struct hash_node *hash_tbl = ss->hash_tbl;
- heap_entry *entry = (heap_entry *)pop_heap(ss->heap);
- if (entry == NULL) {
- return 0;
- }
- unsigned long long count_ret = sorted_set_entry_get_count(entry);
- struct entry_data *tmp_data = sorted_set_entry_get_data(entry);
-
- struct hash_node *s = NULL;
- HASH_FIND_TAG(hash_tbl, tmp_data->key, s);
- if (s == NULL) {
- return 0;
- }
-
- HASH_DEL(hash_tbl, s);
- free(s);
- ss->hash_tbl = hash_tbl;
-
- if (sorted_set_entry_dying(entry)) {
- *popped_cell_id = -1;
- } else {
- *popped_cell_id = tmp_data->cell_id;
- ss->n_living_entry--;
- }
-
- entry_data_destroy(tmp_data);
- free(entry->key);
- free(entry);
-
- return count_ret;
-}
-
-unsigned long long sorted_set_pop(struct sorted_set *ss)
-{
- int dummy;
- unsigned long long count_ret = sorted_set_pop_and_record_popped_back(ss, &dummy);
-
- return count_ret;
-}
-
-bool sorted_set_check_is_full(const struct sorted_set *ss)
-{
- return ss->heap->cur_size >= ss->heap->max_size;
-}
-
-unsigned long long safe_add(unsigned long long a, unsigned long long b)
-{
- if (UINT64_MAX - a < b) {
- return UINT64_MAX;
- }
- return a + b;
-}
-
-void sorted_set_insert_to_available_heap(struct sorted_set *ss, const struct tag_hash_key *tag, unsigned long long cnt, int cell_id)
-{
- cnt = safe_add(cnt, 1); // sorted set will let the count start from 1, 0 for dying entry.
- unsigned long long *tmp_cnt = (unsigned long long*)malloc(sizeof(unsigned long long));
- *tmp_cnt = cnt;
- struct entry_data *tmp_data = entry_data_construct(tag, cell_id);
- heap_entry *node = (heap_entry *)malloc(sizeof(heap_entry));
- node->key = tmp_cnt;
- node->value = tmp_data;
-
- push_heap(ss->heap, (void *)node);
-
- struct hash_node *hash_tbl = ss->hash_tbl;
- struct hash_node *s = (struct hash_node *)malloc(sizeof(struct hash_node));
- s->key = sorted_set_entry_get_data(node)->key;
- s->val = node;
- HASH_ADD_TAG(hash_tbl, key, s);
- ss->hash_tbl = hash_tbl;
-
- ss->n_living_entry++;
-}
-
-int sorted_set_insert(struct sorted_set *ss, const struct tag_hash_key *tag, unsigned long long cnt, int cell_id)
-{
- assert(cell_id >= 0);
- assert(tag != NULL);
- long long tmp_v = tag->hashv;
- assert(tmp_v >= 0);
-
- if (sorted_set_check_is_full(ss)) {
- unsigned long long tmp_mincnt = sorted_set_get_min_count(ss);
- if (tmp_mincnt != INVALID_COUNT && cnt <= tmp_mincnt) { // even if all cells in sorted set are dying, the sorted set can still be a full one, in which case, the min count is invalid.
- return 0;
- }
- sorted_set_pop(ss);
- }
- sorted_set_insert_to_available_heap(ss, tag, cnt, cell_id);
- return 1;
-}
-
-int sorted_set_insert_and_record_popped_back(struct sorted_set *ss, const struct tag_hash_key *tag, unsigned long long cnt, int cell_id, int *popped_user_data)
-{
- assert(cell_id >= 0);
- assert(tag != NULL);
- long long tmp_v = tag->hashv;
- assert(tmp_v >= 0);
-
- // if there is a dying record, reborn it to use.
- heap_entry *entry = sorted_set_find_entry(ss, tag);
- if (entry != NULL) {
- if (!sorted_set_entry_dying(entry)) {
- return -1;
- }
- sorted_set_entry_set_key(ss, entry, safe_add(cnt, 1)); // sorted set will let the count start from 1, 0 for dying entry.
- struct entry_data *data = sorted_set_entry_get_data(entry);
- data->cell_id = cell_id;
- ss->n_living_entry++;
- return 3;
- }
-
- bool popped = false;
- if (sorted_set_check_is_full(ss)) {
- unsigned long long tmp_mincnt = sorted_set_get_min_count(ss);
- if (tmp_mincnt != INVALID_COUNT && cnt <= tmp_mincnt) { // even if all cells in sorted set are dying, the sorted set can still be a full one, in which case, the min count is invalid.
- return 0;
- }
- sorted_set_pop_and_record_popped_back(ss, popped_user_data);
- if (*popped_user_data == -1) { // popped a dying entry, not a real entry.
- popped = false;
- } else {
- popped = true;
- }
- }
- sorted_set_insert_to_available_heap(ss, tag, cnt, cell_id);
- return popped ? 2 : 1;
-}
-
-unsigned long long sorted_set_get_min_count(const struct sorted_set *ss)
-{
- heap *heap = ss->heap;
- if (heap->cur_size == 0) {
- return INVALID_COUNT;
- }
- const heap_entry *ret = (heap_entry *)(heap->nodes[0]);
-
- unsigned long long count = *(unsigned long long *)ret->key;
- if (count == 0) {
- return INVALID_COUNT;
- }
- return count - 1; // sorted set will let the count start from 1, 0 for dying entry.
-}
-
-unsigned long long sorted_set_get_count(const struct sorted_set *ss, const struct tag_hash_key *tag)
-{
- if (sorted_set_cardinality(ss) == 0) {
- return INVALID_COUNT;
- }
- const heap_entry *entry = sorted_set_find_entry(ss, tag);
- if (entry == NULL) {
- return INVALID_COUNT;
- }
- if (sorted_set_entry_dying(entry)) {
- return INVALID_COUNT;
- }
-
- return sorted_set_entry_get_count(entry);
-}
-
-int sorted_set_get_cell_id(const struct sorted_set *ss, const struct tag_hash_key *tag)
-{
- if (sorted_set_cardinality(ss) == 0) {
- return -1;
- }
- const heap_entry *entry = sorted_set_find_entry(ss, tag);
- if (entry == NULL) {
- return -1;
- }
- if (sorted_set_entry_dying(entry)) {
- return -1;
- }
-
- return sorted_set_entry_get_data(entry)->cell_id;
-}
-
-int sorted_set_cardinality(const struct sorted_set *ss)
-{
- return ss->n_living_entry;
-}
-
-int sorted_set_incrby(struct sorted_set *ss, const struct tag_hash_key *tag, unsigned long long count)
-{
- heap_entry *entry = sorted_set_find_entry(ss, tag);
- if (entry == NULL) {
- return -1;
- }
- unsigned long long cnt_old;
- if (sorted_set_entry_dying(entry) == false) {
- cnt_old = sorted_set_entry_get_count(entry);
- cnt_old += 1; // sorted set will let the count start from 1, 0 for dying entry.
- } else {
- cnt_old = 0;
- }
- sorted_set_entry_set_key(ss, entry, safe_add(count, cnt_old));
- return 0;
-}
-
-typedef struct {
- unsigned long long key;
- struct entry_data *val;
-} tmp_heap_node;
-
-int cmp_tmp_heap_node(const void *a, const void *b)
-{
- const tmp_heap_node *aa = (tmp_heap_node *)a;
- const tmp_heap_node *bb = (tmp_heap_node *)b;
-
- if (aa->key < bb->key) {
- return 1;
- } else if (aa->key > bb->key) {
- return -1;
- } else {
- return 0;
- }
-}
-
-void sorted_set_dump(const struct sorted_set *ss, unsigned long long *scores_out, struct tag_hash_key **tags_out, int *user_data_out)
-{
- struct minheap *h = ss->heap;
- tmp_heap_node *tmp_nodes = (tmp_heap_node *)malloc(sizeof(tmp_heap_node) * h->cur_size);
- size_t n_living_entry = 0;
- for (int i = 0; i < h->cur_size; i++) {
- const heap_entry *entry = (heap_entry *)h->nodes[i];
- if (sorted_set_entry_dying(entry)) {
- continue;
- }
- tmp_nodes[n_living_entry].key = sorted_set_entry_get_count(entry);
- tmp_nodes[n_living_entry].val = sorted_set_entry_get_data(entry);
- n_living_entry++;
- }
- assert(n_living_entry == ss->n_living_entry);
- qsort(tmp_nodes, n_living_entry, sizeof(tmp_heap_node), cmp_tmp_heap_node);
- for (int i = 0; i < n_living_entry; i++) {
- assert(tmp_nodes[i].val != NULL);
- assert(tmp_nodes[i].val->cell_id >= 0);
- assert(tmp_nodes[i].val->key != NULL);
- long long tmp_haha = tmp_nodes[i].val->key->hashv;
- assert(tmp_haha >= 0);
-
- scores_out[i] = tmp_nodes[i].key;
- tags_out[i] = tmp_nodes[i].val->key;
- user_data_out[i] = tmp_nodes[i].val->cell_id;
- }
- free(tmp_nodes);
-}
-
-struct sorted_set *sorted_set_copy(const struct sorted_set *ss)
-{
- struct sorted_set *ret = sorted_set_new(ss->heap->max_size);
-
- for (int i = 0; i < ss->heap->cur_size; i++) {
- const heap_entry *entry = (heap_entry *)ss->heap->nodes[i];
- if (sorted_set_entry_dying(entry)) {
- continue;
- }
- const struct entry_data *data = sorted_set_entry_get_data(entry);
- sorted_set_insert(ret, data->key, sorted_set_entry_get_count(entry), data->cell_id);
- }
-
- return ret;
-}
-
-// set all the entry to dying status. They will be the first to be popped.
-void sorted_set_reset(struct sorted_set *ss)
-{
- struct hash_node *tbl = ss->hash_tbl;
- struct hash_node *h_node, *tmp;
- HASH_ITER(hh, tbl, h_node, tmp) {
- heap_entry *entry = (heap_entry *)h_node->val;
- *(unsigned long long *)entry->key = 0;
- }
-
- heap *heap = ss->heap;
- heap_entry *entry;
- for (int i = 0; i < heap->cur_size; i++) {
- entry = (heap_entry *)heap->nodes[i];
- *(unsigned long long *)entry->key = 0;
- }
-
- ss->n_living_entry = 0;
-}
-
-struct tag_hash_key *sorted_set_get_true_pst_tag(const struct sorted_set *ss, const struct tag_hash_key *tag)
-{
- const heap_entry *entry = sorted_set_find_entry(ss, tag);
- if (entry == NULL) {
- return NULL;
- }
- if (sorted_set_entry_dying(entry)) {
- return NULL;
- }
- const struct entry_data *data = sorted_set_entry_get_data(entry);
- return data->key;
-}
\ No newline at end of file diff --git a/src/tags/sorted_set.h b/src/tags/sorted_set.h deleted file mode 100644 index 6dee8c7..0000000 --- a/src/tags/sorted_set.h +++ /dev/null @@ -1,89 +0,0 @@ -#pragma once
-
-#include <stddef.h>
-#include "my_ut_hash.h"
-
-#ifdef __cplusplus
-extern "C"{
-#endif
-
-#define INVALID_COUNT UINT64_MAX
-
-struct sorted_set;
-
-/**
- * @brief Create a new sorted set.
- * @param set_size the max size of the sorted set.
- * @return the pointer to the sorted set.
-*/
-struct sorted_set *sorted_set_new(int set_size);
-
-/**
- * @brief Destroy the sorted set.
- * @param ss the pointer to the sorted set
-*/
-void sorted_set_free(struct sorted_set *ss);
-
-
-/**
- * @brief insert a new entry into the sorted set. The minimum entry will be popped if the list is full.
- * @param tag the pointer to the tag, user should alloc it, when the entry is popped out, the tag will be freed. If the tag is not inserted to the sorted set, user should free it.
- * @return 0 if the entry is not inserted because the sorted set is full and the score of the entry is less than the minimum score in the sorted set. Otherwise, return 1.
-*/
-int sorted_set_insert(struct sorted_set *ss, const struct tag_hash_key *tag, unsigned long long cnt, int cell_id);
-
-/*
- @return -1 if the entry already exist. 0 if the entry is not inserted because the sorted set is full. return 2 if some data is popped out. Otherwise, return 1 if the entry is inserted successfully and sorted set not full.
- @param popped_user_data user should alloc it, popped_user_data_size is equal to the size of the user data memory.
-*/
-int sorted_set_insert_and_record_popped_back(struct sorted_set *ss, const struct tag_hash_key *tag, unsigned long long cnt, int cell_id, int *popped_user_data);
-
-/**
- * @brief pop out the minumum-score entry from the sorted set.
- * @param ss the sorted set
- * @return the score of the minimum-score member. return 0 if the sorted set is empty.
-*/
-unsigned long long sorted_set_pop(struct sorted_set *ss);
-
-/**
- * @brief Increase the score of the given member.
- * @param score By the value of which increase member.
-*/
-int sorted_set_incrby(struct sorted_set *ss, const struct tag_hash_key *tag, unsigned long long count);
-
-/**
- * @brief get the minimum score from the sorted set.
- * @param ss the sorted set
- * @return the score of the minimum-score member
-*/
-unsigned long long sorted_set_get_min_count(const struct sorted_set *ss);
-
-/**
- * @brief find the how many elements the sorted set contains.
-*/
-int sorted_set_cardinality(const struct sorted_set *ss);
-
-/**
- * @brief Get the score of the given key string.
- * @return the score of the member. return 0 if the member is not in the sorted set.
-*/
-unsigned long long sorted_set_get_count(const struct sorted_set *ss, const struct tag_hash_key *tag);
-int sorted_set_get_cell_id(const struct sorted_set *ss, const struct tag_hash_key *tag);
-/**
- * @brief Get all the contents in sorted order from small to big.
- * @param scores_out the scores of the entries. User should alloc it and make sure the buffer is long enough.
- * @param tags_out the tags of the entries. User should alloc it and make sure the buffer is long enough. Don't free the content.
- * @param user_data_out the user data of the entries. User should alloc it and make sure the buffer is long enough. Its content is the pointer to the user data in the sorted set. Don't free the content.
-*/
-void sorted_set_dump(const struct sorted_set *ss, unsigned long long *scores_out, struct tag_hash_key **tags_out, int *user_data_out);
-
-struct sorted_set *sorted_set_copy(const struct sorted_set *ss);
-
-void sorted_set_reset(struct sorted_set *ss);
-
-// the cell_manager id_tag_map will not stort the copied tag, instead, it will store the pointer to the tag in the sorted set. Use this function to give it one.
-struct tag_hash_key *sorted_set_get_true_pst_tag(const struct sorted_set *ss, const struct tag_hash_key *tag);
-
-#ifdef __cplusplus
-}//end extern "C"
-#endif
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 61c6b75..a6c1403 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -16,7 +16,7 @@ set(DEBUG_FLAGS "-O3") include_directories(${PROJECT_SOURCE_DIR}/test)
include_directories(${PROJECT_SOURCE_DIR}/test/deps)
include_directories(${PROJECT_SOURCE_DIR}/src)
-include_directories(${PROJECT_SOURCE_DIR}/src/tags)
+include_directories(${PROJECT_SOURCE_DIR}/src/cells)
include_directories(${PROJECT_SOURCE_DIR}/src/metrics)
include_directories(${PROJECT_SOURCE_DIR}/include/fieldstat)
@@ -57,5 +57,4 @@ add_unit_test(test_metric_histogram) add_unit_test(test_metric_hll)
add_unit_test(test_performance)
add_unit_test(test_register_and_reset)
-add_unit_test(unit_test_cell_manager)
add_unit_test(test_write_json_file)
\ No newline at end of file diff --git a/test/profiling/CMakeLists.txt b/test/profiling/CMakeLists.txt index 5c8e99c..f06242f 100644 --- a/test/profiling/CMakeLists.txt +++ b/test/profiling/CMakeLists.txt @@ -16,7 +16,7 @@ include_directories(${PROJECT_SOURCE_DIR}/src/utils) file(GLOB SRC
"${PROJECT_SOURCE_DIR}/src/*.c"
"${PROJECT_SOURCE_DIR}/src/metrics/*.c"
- "${PROJECT_SOURCE_DIR}/src/tags/*.c"
+ "${PROJECT_SOURCE_DIR}/src/cells/*.c"
"${PROJECT_SOURCE_DIR}/src/exporter/*.c"
"${PROJECT_SOURCE_DIR}/src/utils/*.c"
"${PROJECT_SOURCE_DIR}/vendors/cjson/*.c"
diff --git a/test/profiling/main.c b/test/profiling/main.c index caf6542..9719579 100644 --- a/test/profiling/main.c +++ b/test/profiling/main.c @@ -4,36 +4,34 @@ #include <string.h> #include "fieldstat.h" -#include "tags/my_ut_hash.h" -#include "tags/my_ut_hash_inner.h" +#include "fields/my_ut_hash.h" +#include "fields/my_ut_hash_inner.h" #define ADD_OPER_NUM 5000000ULL // #define ADD_OPER_NUM 1 #define MAX_STRING_KEY_LEN 10 -const struct fieldstat_tag TEST_TAG_INT = {"INT key_", TAG_INTEGER, {.value_longlong = 100}}; +const struct field TEST_FIELD_INT = {"INT key_", FIELD_VALUE_INTEGER, {.value_longlong = 100}}; -// todo: 问一下贺岚风，tag是什么样子的 - int main () { printf("Start profiling...\n"); clock_t start, end; - // struct fieldstat_tag TAG[4] = { - // {"INT key_", TAG_INTEGER, {.value_longlong = 100}}, - // {"STRING key_", TAG_CSTRING, {.value_str = "10012312312312"}}, - // {"STRING key_", TAG_CSTRING, {.value_str = "100adsffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}}, - // {"FLOAT key_", TAG_INTEGER, {.value_double = 100.0}}, + // struct field TAG[4] = { + // {"INT key_", FIELD_VALUE_INTEGER, {.value_longlong = 100}}, + // {"STRING key_", FIELD_VALUE_CSTRING, {.value_str = "10012312312312"}}, + // {"STRING key_", FIELD_VALUE_CSTRING, {.value_str = "100adsffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}}, + // {"FLOAT key_", FIELD_VALUE_INTEGER, {.value_double = 100.0}}, // }; - struct fieldstat_tag TAG[4] = { - {"object_id", TAG_INTEGER, {.value_longlong = 20}}, - {"item_id", TAG_INTEGER, {.value_longlong = 16916397}}, - {"chart_id", TAG_INTEGER, {.value_longlong = 1}}, - {"version", TAG_INTEGER, {.value_longlong = 1}}, + struct field TAG[4] = { + {"object_id", FIELD_VALUE_INTEGER, {.value_longlong = 20}}, + {"item_id", FIELD_VALUE_INTEGER, {.value_longlong = 16916397}}, + {"chart_id", FIELD_VALUE_INTEGER, {.value_longlong = 1}}, + {"version", FIELD_VALUE_INTEGER, {.value_longlong = 1}}, }; struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); + fieldstat_cube_create(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); fieldstat_register_counter(instance, "counter"); start = clock(); diff --git a/test/test_easy_fs.cpp b/test/test_easy_fs.cpp index d3f45b7..a0e507d 100644 --- a/test/test_easy_fs.cpp +++ b/test/test_easy_fs.cpp @@ -13,54 +13,25 @@ TEST(test_easy_fieldstat, new_and_free) { - struct fieldstat_easy *fse = fieldstat_easy_new(10, "instance name1231231231", &TEST_TAG_STRING, 1); + struct fieldstat_easy *fse = fieldstat_easy_new(10, "instance name1231231231", &TEST_FIELD_STRING, 1); fieldstat_easy_free(fse); } TEST(test_easy_fieldstat, output_to_buff) { - struct fieldstat_easy *fse = fieldstat_easy_new(10, NULL, &TEST_TAG_STRING, 1); + struct fieldstat_easy *fse = fieldstat_easy_new(10, NULL, &TEST_FIELD_STRING, 1); fieldstat_easy_register_counter(fse, "metric counter"); - fieldstat_easy_counter_incrby(fse, 0, 0, &TEST_TAG_INT, 1, 1); + fieldstat_easy_counter_incrby(fse, 0, 0, &TEST_FIELD_INT, 1, 1); // read file, should be empty char *buff = NULL; size_t buff_len = 0; fieldstat_easy_output(fse, &buff, &buff_len); cJSON *root = cJSON_Parse(buff); cJSON *cell = cJSON_GetArrayItem(root, 0); - cJSON *metric = cJSON_GetObjectItem(cell, "fields"); - long long value = cJSON_GetObjectItem(metric, "metric counter")->valueint; + long long value = cJSON_GetObjectItem(cell, "metric counter")->valueint; EXPECT_EQ(value, 1); - cJSON *tags = cJSON_GetObjectItem(cell, "tags"); - EXPECT_EQ(cJSON_GetObjectItem(tags, TEST_TAG_INT.key)->valueint, TEST_TAG_INT.value_longlong); - EXPECT_STREQ(cJSON_GetObjectItem(tags, TEST_TAG_STRING.key)->valuestring, TEST_TAG_STRING.value_str); - - cJSON_Delete(root); - free(buff); - fieldstat_easy_free(fse); -} - -TEST(test_easy_fieldstat, output_to_buff_with_delta) -{ - struct fieldstat_easy *fse = fieldstat_easy_new(1, NULL, NULL, 0); - fieldstat_easy_enable_delta_in_active_output(fse); - fieldstat_easy_register_counter(fse, "metric counter"); - fieldstat_easy_counter_incrby(fse, 0, 0, NULL, 0, 1); - char *buff = NULL; - size_t buff_len = 0; - fieldstat_easy_output(fse, &buff, &buff_len); - free(buff); - fieldstat_easy_counter_incrby(fse, 0, 0, NULL, 0, 1); - fieldstat_easy_output(fse, &buff, &buff_len); - - cJSON *root = cJSON_Parse(buff); - cJSON *cell = cJSON_GetArrayItem(root, 0); - cJSON *metric = cJSON_GetObjectItem(cell, "fields"); - long long value = cJSON_GetObjectItem(metric, "metric counter")->valueint; - EXPECT_EQ(value, 2); - cJSON *metric_delta = cJSON_GetObjectItem(cell, "fields_delta"); - long long value_delta = cJSON_GetObjectItem(metric_delta, "metric counter")->valueint; - EXPECT_EQ(value_delta, 1); + EXPECT_EQ(cJSON_GetObjectItem(cell, TEST_FIELD_INT.key)->valueint, TEST_FIELD_INT.value_longlong); + EXPECT_STREQ(cJSON_GetObjectItem(cell, TEST_FIELD_STRING.key)->valuestring, TEST_FIELD_STRING.value_str); cJSON_Delete(root); free(buff); @@ -92,9 +63,9 @@ TEST(test_easy_fieldstat, output_to_file) EXPECT_EQ(cJSON_GetArraySize(root), 0); // is empty and valid cJSON_Delete(root); - fieldstat_easy_counter_incrby(fse, 0, counter_id, &TEST_TAG_INT, 1, 1); - fieldstat_easy_counter_incrby(fse, 1, counter_id, &TEST_TAG_INT, 1, 10); - fieldstat_easy_counter_incrby(fse, 2, counter_id, &TEST_TAG_INT, 1, 100); + fieldstat_easy_counter_incrby(fse, 0, counter_id, &TEST_FIELD_INT, 1, 1); + fieldstat_easy_counter_incrby(fse, 1, counter_id, &TEST_FIELD_INT, 1, 10); + fieldstat_easy_counter_incrby(fse, 2, counter_id, &TEST_FIELD_INT, 1, 100); sleep(2); // 2nd interval: merge 3 thread's data, and output @@ -123,7 +94,7 @@ TEST(test_easy_fieldstat, output_to_file) cJSON_Delete(root); // 4th interval: new data, output again - fieldstat_easy_counter_incrby(fse, 0, counter_id, &TEST_TAG_DOUBLE, 1, 10086); + fieldstat_easy_counter_incrby(fse, 0, counter_id, &TEST_FIELD_DOUBLE, 1, 10086); sleep(2); printf("4th interval\n"); root = read_file(); @@ -182,9 +153,9 @@ TEST(test_easy_fieldstat, output_interval_ok) struct fieldstat_easy *fse = fieldstat_easy_new(10, NULL, NULL, 0); fieldstat_easy_register_histogram(fse, "metric histogram", 1, 10000, 3); // a pretty time consuming metric - fieldstat_easy_histogram_record(fse, 0, 0, &TEST_TAG_INT, 1, 1); - fieldstat_easy_histogram_record(fse, 0, 0, &TEST_TAG_DOUBLE, 1, 10); - fieldstat_easy_histogram_record(fse, 0, 0, &TEST_TAG_STRING, 1, 110); + fieldstat_easy_histogram_record(fse, 0, 0, &TEST_FIELD_INT, 1, 1); + fieldstat_easy_histogram_record(fse, 0, 0, &TEST_FIELD_DOUBLE, 1, 10); + fieldstat_easy_histogram_record(fse, 0, 0, &TEST_FIELD_STRING, 1, 110); fieldstat_easy_enable_auto_output(fse, FILENAME, 1); @@ -197,10 +168,10 @@ TEST(test_easy_fieldstat, output_interval_ok) TEST(test_easy_fieldstat, ensure_data_racing_of_two_output_and_of_incyby) { const int N_THREADS = 3; - struct fieldstat_tag global_tags[1]; - struct fieldstat_tag tmptag; + struct field global_tags[1]; + struct field tmptag; tmptag.key = "app id"; - tmptag.type = TAG_INTEGER; + tmptag.type = FIELD_VALUE_INTEGER; tmptag.value_longlong = 1; global_tags[0] = tmptag; @@ -213,8 +184,8 @@ TEST(test_easy_fieldstat, ensure_data_racing_of_two_output_and_of_incyby) for (int thread_id = 0; thread_id < N_THREADS; thread_id++) { threads[thread_id] = std::thread([fse, counter_id, hdr_id, thread_id]() { for (int i = 0; i < 1000000; i++) { // loop million times to ensure no core dump - fieldstat_easy_counter_incrby(fse, thread_id, counter_id, &TEST_TAG_INT, 1, 1); - fieldstat_easy_histogram_record(fse, thread_id, hdr_id, &TEST_TAG_INT, 1, rand() % 10000); + fieldstat_easy_counter_incrby(fse, thread_id, counter_id, &TEST_FIELD_INT, 1, 1); + fieldstat_easy_histogram_record(fse, thread_id, hdr_id, &TEST_FIELD_INT, 1, rand() % 10000); usleep(1); // 1us * 1000000 = 1s, just long enough to output } }); @@ -231,8 +202,8 @@ TEST(test_easy_fieldstat, reset) { struct fieldstat_easy *fse = fieldstat_easy_new(3, NULL, NULL, 0); int counter_id = fieldstat_easy_register_counter(fse, "metric counter"); - fieldstat_easy_counter_incrby(fse, 0, counter_id, &TEST_TAG_INT, 1, 1); - fieldstat_easy_counter_incrby(fse, 1, counter_id, &TEST_TAG_INT, 1, 1); + fieldstat_easy_counter_incrby(fse, 0, counter_id, &TEST_FIELD_INT, 1, 1); + fieldstat_easy_counter_incrby(fse, 1, counter_id, &TEST_FIELD_INT, 1, 1); char **objects = NULL; size_t n_objects = 0; @@ -250,8 +221,7 @@ TEST(test_easy_fieldstat, reset) long long get_value(const char *input, const char *key) { cJSON *root = cJSON_Parse(input); - cJSON *metric = cJSON_GetObjectItem(root, "fields"); - long long value = cJSON_GetObjectItem(metric, key)->valueint; + long long value = cJSON_GetObjectItem(root, key)->valueint; cJSON_Delete(root); return value; } @@ -268,7 +238,7 @@ TEST(test_easy_fieldstat, accuracy_in_multithread) for (int thread_id = 0; thread_id < N_THREADS; thread_id++) { threads[thread_id] = std::thread([fse, counter_id, thread_id]() { for (size_t i = 0; i < 2000000ULL; i++) { // 1 million times - fieldstat_easy_counter_incrby(fse, thread_id, counter_id, &TEST_TAG_INT, 1, 1); + fieldstat_easy_counter_incrby(fse, thread_id, counter_id, &TEST_FIELD_INT, 1, 1); } }); } @@ -305,5 +275,6 @@ TEST(test_easy_fieldstat, accuracy_in_multithread) int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); + // testing::GTEST_FLAG(filter) = "test_easy_fieldstat.accuracy_in_multithread"; return RUN_ALL_TESTS(); }
\ No newline at end of file diff --git a/test/test_empty_tags.cpp b/test/test_empty_tags.cpp index 4f500bd..9cbaead 100644 --- a/test/test_empty_tags.cpp +++ b/test/test_empty_tags.cpp @@ -8,25 +8,25 @@ void assert_cell_null(const struct fieldstat *instance, int cube_id, long long expected_count) { - struct fieldstat_tag_list *tag_list = NULL; + struct field_list *tag_list = NULL; size_t n_cell = 0; - fieldstat_get_cells_used_by_cube(instance, cube_id, &tag_list, &n_cell); + fieldstat_cube_get_cells(instance, cube_id, &tag_list, &n_cell); EXPECT_EQ(n_cell, 1); - EXPECT_EQ(tag_list[0].n_tag, 0); - EXPECT_TRUE(tag_list[0].tag == NULL); + EXPECT_EQ(tag_list[0].n_field, 0); + EXPECT_TRUE(tag_list[0].field == NULL); long long value; - fieldstat_counter_get(instance, cube_id, 0, &tag_list[0], &value); + fieldstat_counter_get(instance, cube_id, &tag_list[0], 0, &value); EXPECT_EQ(value, expected_count); - fieldstat_tag_list_arr_free(tag_list, n_cell); + fieldstat_field_list_arr_free(tag_list, n_cell); } - TEST(test_empty_tag, add_many_times) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 1); + int cube_id = fieldstat_cube_create(instance, NULL, 0); + int metric_id = fieldstat_register_counter(instance, cube_id, "metric"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_COMPREHENSIVE, 1, 0); - int metric_id = fieldstat_register_counter(instance, "metric"); fieldstat_counter_incrby(instance, cube_id, metric_id, NULL, 0, 1); fieldstat_counter_incrby(instance, cube_id, metric_id, NULL, 0, 1); @@ -38,8 +38,10 @@ TEST(test_empty_tag, add_many_times) struct fieldstat *test_empty_my_init(enum sampling_mode mode = SAMPLING_MODE_COMPREHENSIVE) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, NULL, 0, mode, 1); - int metric_id = fieldstat_register_counter(instance, "metric"); + int cube_id = fieldstat_cube_create(instance, NULL, 0); + int metric_id = fieldstat_register_counter(instance, cube_id, "metric"); + fieldstat_cube_set_sampling(instance, cube_id, mode, 1, 0); + fieldstat_counter_incrby(instance, cube_id, metric_id, NULL, 0, 1); return instance; @@ -57,10 +59,10 @@ TEST(test_empty_tag, merge) int n_cube = 0; fieldstat_get_cubes(instance_dst, &ret_cube_id_arr, &n_cube); int ret_cell_id = ret_cube_id_arr[0]; - struct fieldstat_tag_list *shared_tag = fieldstat_get_shared_tags(instance_dst, ret_cell_id); - EXPECT_EQ(shared_tag->n_tag, 0); - EXPECT_TRUE(shared_tag->tag == NULL); - fieldstat_tag_list_arr_free(shared_tag, 1); + struct field_list *shared_tag = fieldstat_cube_get_dimensions(instance_dst, ret_cell_id); + EXPECT_EQ(shared_tag->n_field, 0); + EXPECT_TRUE(shared_tag->field == NULL); + fieldstat_field_list_arr_free(shared_tag, 1); free(ret_cube_id_arr); assert_cell_null(instance_dst, ret_cell_id, 2); @@ -81,10 +83,10 @@ TEST(test_empty_tag, merge_topk) int n_cube = 0; fieldstat_get_cubes(instance_dst, &ret_cube_id_arr, &n_cube); int ret_cell_id = ret_cube_id_arr[0]; - struct fieldstat_tag_list *shared_tag = fieldstat_get_shared_tags(instance_dst, ret_cell_id); - EXPECT_EQ(shared_tag->n_tag, 0); - EXPECT_TRUE(shared_tag->tag == NULL); - fieldstat_tag_list_arr_free(shared_tag, 1); + struct field_list *shared_tag = fieldstat_cube_get_dimensions(instance_dst, ret_cell_id); + EXPECT_EQ(shared_tag->n_field, 0); + EXPECT_TRUE(shared_tag->field == NULL); + fieldstat_field_list_arr_free(shared_tag, 1); free(ret_cube_id_arr); assert_cell_null(instance_dst, ret_cell_id, 2); @@ -94,6 +96,43 @@ TEST(test_empty_tag, merge_topk) fieldstat_free(instance_src); } +TEST(test_empty_tag, merge_spreadsketch) +{ + struct fieldstat *instance_src = fieldstat_new(); + int cube_id = fieldstat_cube_create(instance_src, NULL, 0); + int metric_id = fieldstat_register_hll(instance_src, cube_id, "metric", 4); + fieldstat_cube_set_sampling(instance_src, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 1, 0); + fieldstat_hll_add(instance_src, cube_id, metric_id, NULL, 0, "1", 1); + struct fieldstat *instance_dst = fieldstat_new(); + + fieldstat_merge(instance_dst, instance_src); + fieldstat_merge(instance_dst, instance_src); + + int *ret_cube_id_arr = NULL; + int n_cube = 0; + fieldstat_get_cubes(instance_dst, &ret_cube_id_arr, &n_cube); + int ret_cell_id = ret_cube_id_arr[0]; + struct field_list *shared_tag = fieldstat_cube_get_dimensions(instance_dst, ret_cell_id); + EXPECT_EQ(shared_tag->n_field, 0); + EXPECT_TRUE(shared_tag->field == NULL); + fieldstat_field_list_arr_free(shared_tag, 1); + free(ret_cube_id_arr); + + struct field_list *tag_list = NULL; + size_t n_cell = 0; + fieldstat_cube_get_cells(instance_dst, cube_id, &tag_list, &n_cell); + EXPECT_EQ(n_cell, 1); + EXPECT_EQ(tag_list[0].n_field, 0); + EXPECT_TRUE(tag_list[0].field == NULL); + double value; + fieldstat_hll_get(instance_dst, cube_id, &tag_list[0], 0, &value); + EXPECT_NEAR(value, 1, 0.4); + fieldstat_field_list_arr_free(tag_list, n_cell); + + fieldstat_free(instance_dst); + fieldstat_free(instance_src); +} + TEST(test_empty_tag, export) { struct fieldstat *instance = test_empty_my_init(); @@ -107,9 +146,9 @@ TEST(test_empty_tag, export) fieldstat_free(instance); cJSON *root = cJSON_GetArrayItem(root_arr, 0); - // check tag - cJSON *tag = cJSON_GetObjectItem(root, "tags"); - EXPECT_EQ(cJSON_GetArraySize(tag), 0); + // check field + cJSON *field = cJSON_GetObjectItem(root, "tags"); + EXPECT_EQ(cJSON_GetArraySize(field), 0); cJSON *metrics = cJSON_GetObjectItem(root, "fields"); EXPECT_NE(metrics, nullptr); diff --git a/test/test_exporter_json.cpp b/test/test_exporter_json.cpp index 09aa579..f405b7f 100644 --- a/test/test_exporter_json.cpp +++ b/test/test_exporter_json.cpp @@ -1,6 +1,7 @@ #include <gtest/gtest.h> #include <random> +#include <unordered_set> #include "cjson/cJSON.h" #include "base64/b64.h" @@ -10,35 +11,35 @@ #include "utils.hpp" #include "hdr/hdr_histogram.h" #include "hdr/hdr_histogram_log.h" -#include "st_hyperloglog.h" +#include "hyperloglog.h" #include "histogram_encoder.h" #include "very_fast_json_writer.h" const size_t OPER_NUM = 10000; std::string g_hll_standard_oper[OPER_NUM]; long long g_histogram_standard_oper[OPER_NUM]; -struct ST_hyperloglog *g_hll_standard; +struct hyperloglog *g_hll_standard; struct hdr_histogram *g_histogram_standard; #define TEST_TOPK_STANDARD_K 5 #define TEST_METRIC_NUM 2 -const struct fieldstat_tag TEST_TAG_GLOBAL1 = {"test_tag_global 1", .type = TAG_INTEGER, {.value_longlong = 1}}; -const struct fieldstat_tag TEST_TAG_GLOBAL2 = {"test_tag_global 2", .type = TAG_DOUBLE, {.value_double = 2.2}}; -const struct fieldstat_tag TEST_TAG_GLOBAL3 = {"test_tag_global 3", .type = TAG_CSTRING, {.value_str = "string3"}}; -const struct fieldstat_tag TEST_TAG_GLOBAL[3] = {TEST_TAG_GLOBAL1, TEST_TAG_GLOBAL2, TEST_TAG_GLOBAL3}; +const struct field TEST_TAG_GLOBAL1 = {"test_tag_global 1", .type = FIELD_VALUE_INTEGER, {.value_longlong = 1}}; +const struct field TEST_TAG_GLOBAL2 = {"test_tag_global 2", .type = FIELD_VALUE_DOUBLE, {.value_double = 2.2}}; +const struct field TEST_TAG_GLOBAL3 = {"test_tag_global 3", .type = FIELD_VALUE_CSTRING, {.value_str = "string3"}}; +const struct field TEST_TAG_GLOBAL[3] = {TEST_TAG_GLOBAL1, TEST_TAG_GLOBAL2, TEST_TAG_GLOBAL3}; -const struct fieldstat_tag TEST_TAG_SHARED1_1 = {"test_tag_shared 1", .type = TAG_INTEGER, {.value_longlong = 3}}; -const struct fieldstat_tag TEST_TAG_SHARED1_2 = {"test_tag_shared 2", .type = TAG_DOUBLE, {.value_double = 0.2}}; -const struct fieldstat_tag TEST_TAG_SHARED1_3 = {"test_tag_shared 3", .type = TAG_CSTRING, {.value_str = "1string"}}; -const struct fieldstat_tag TEST_TAG_SHARED1[3] = {TEST_TAG_SHARED1_1, TEST_TAG_SHARED1_2, TEST_TAG_SHARED1_3}; +const struct field TEST_TAG_SHARED1_1 = {"test_tag_shared 1", .type = FIELD_VALUE_INTEGER, {.value_longlong = 3}}; +const struct field TEST_TAG_SHARED1_2 = {"test_tag_shared 2", .type = FIELD_VALUE_DOUBLE, {.value_double = 0.2}}; +const struct field TEST_TAG_SHARED1_3 = {"test_tag_shared 3", .type = FIELD_VALUE_CSTRING, {.value_str = "1string"}}; +const struct field TEST_TAG_SHARED1[3] = {TEST_TAG_SHARED1_1, TEST_TAG_SHARED1_2, TEST_TAG_SHARED1_3}; -const struct fieldstat_tag TEST_TAG_SHARED2_1 = {"test_tag_shared 11", .type = TAG_INTEGER, {.value_longlong = 4}}; -const struct fieldstat_tag TEST_TAG_SHARED2_2 = {"test_tag_shared 22", .type = TAG_DOUBLE, {.value_double = 0.3}}; -const struct fieldstat_tag TEST_TAG_SHARED2_3 = {"test_tag_shared 33", .type = TAG_CSTRING, {.value_str = "2string"}}; -const struct fieldstat_tag TEST_TAG_SHARED2[3] = {TEST_TAG_SHARED2_1, TEST_TAG_SHARED2_2, TEST_TAG_SHARED2_3}; +const struct field TEST_TAG_SHARED2_1 = {"test_tag_shared 11", .type = FIELD_VALUE_INTEGER, {.value_longlong = 4}}; +const struct field TEST_TAG_SHARED2_2 = {"test_tag_shared 22", .type = FIELD_VALUE_DOUBLE, {.value_double = 0.3}}; +const struct field TEST_TAG_SHARED2_3 = {"test_tag_shared 33", .type = FIELD_VALUE_CSTRING, {.value_str = "2string"}}; +const struct field TEST_TAG_SHARED2[3] = {TEST_TAG_SHARED2_1, TEST_TAG_SHARED2_2, TEST_TAG_SHARED2_3}; -const struct fieldstat_tag TEST_TAG_SHARED3_1 = {"test_tag_shared 3", .type = TAG_INTEGER, {.value_longlong = 5}}; -const struct fieldstat_tag TEST_TAG_SHARED3[1] = {TEST_TAG_SHARED3_1}; +const struct field TEST_TAG_SHARED3_1 = {"test_tag_shared 3", .type = FIELD_VALUE_INTEGER, {.value_longlong = 5}}; +const struct field TEST_TAG_SHARED3[1] = {TEST_TAG_SHARED3_1}; void test_check_if_tag_list_is_in_json(cJSON *tag_obj, const Fieldstat_tag_list_wrapper *benchmark) { @@ -48,7 +49,7 @@ void test_check_if_tag_list_is_in_json(cJSON *tag_obj, const Fieldstat_tag_list_ if (tag_val->type == cJSON_String) { EXPECT_STREQ(tag_val->valuestring, benchmark->get_tag()[tag_id].value_str); } else { - if (benchmark->get_tag()[tag_id].type == TAG_INTEGER) { + if (benchmark->get_tag()[tag_id].type == FIELD_VALUE_INTEGER) { EXPECT_EQ(tag_val->valueint, benchmark->get_tag()[tag_id].value_longlong); } else { EXPECT_NEAR(tag_val->valuedouble, benchmark->get_tag()[tag_id].value_double, 0.0001); @@ -59,7 +60,7 @@ void test_check_if_tag_list_is_in_json(cJSON *tag_obj, const Fieldstat_tag_list_ void test_check_if_global_tag_is_in_json(cJSON *tag_obj) { - struct fieldstat_tag_list tag_list = {.tag = (struct fieldstat_tag *)TEST_TAG_GLOBAL, .n_tag = 3}; + struct field_list tag_list = {.field = (struct field *)TEST_TAG_GLOBAL, .n_field = 3}; const Fieldstat_tag_list_wrapper benchmark(&tag_list); test_check_if_tag_list_is_in_json(tag_obj, &benchmark); } @@ -83,7 +84,7 @@ void test_check_if_metric_gauge_correct(cJSON *metric_obj, const char *name) { char *blob_gauge_benchmark = NULL; size_t size_dummy = 0; - ST_hyperloglog_serialize_for_networking(g_hll_standard, &blob_gauge_benchmark, &size_dummy); + hyperloglog_serialize_into_base64(g_hll_standard, &blob_gauge_benchmark, &size_dummy); cJSON *gauge_obj = cJSON_GetObjectItem(metric_obj, name); EXPECT_NE(gauge_obj, nullptr); @@ -104,7 +105,7 @@ void test_check_if_metric_topk_correct(cJSON *metric_obj, const char **name, uns } } -void fill_random_tag(Fieldstat_tag_list_wrapper *tags[], int tag_list_num) +void fill_random_tag(Fieldstat_tag_list_wrapper *fields[], int tag_list_num) { std::uniform_int_distribution<int> dist(1,100); std::mt19937 rng(); @@ -112,7 +113,7 @@ void fill_random_tag(Fieldstat_tag_list_wrapper *tags[], int tag_list_num) for (int i = 0; i < tag_list_num; i++) { Fieldstat_tag_list_wrapper *tmp = new Fieldstat_tag_list_wrapper(dist, i + 1); - tags[i] = tmp; + fields[i] = tmp; } } @@ -166,9 +167,10 @@ void topk_standard_oper(const std::function<void(Fieldstat_tag_list_wrapper *, u void topk_init(struct fieldstat *instance, unsigned int test_expected_big_count) { const char *metric_name[TEST_METRIC_NUM] = {"topk1", "topk2"}; - int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_TOPK, TEST_TOPK_STANDARD_K); - int m1 = fieldstat_register_counter(instance, metric_name[0]); - int m2 = fieldstat_register_counter(instance, metric_name[1]); + int cube_id = fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3); + int m1 = fieldstat_register_counter(instance, cube_id, metric_name[0]); + int m2 = fieldstat_register_counter(instance, cube_id, metric_name[1]); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, TEST_TOPK_STANDARD_K, 0); std::function<void(Fieldstat_tag_list_wrapper *, unsigned int *)> topk_add = [instance, cube_id, m1, m2](const Fieldstat_tag_list_wrapper *my_tags, unsigned int counts[TEST_METRIC_NUM]) { @@ -185,23 +187,25 @@ TEST(export_test, cjson_export_with_fixed_tag_and_many_metrics_on_one_cube_of_co // new instance struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, tag_list_num); - int id_counter = fieldstat_register_counter(instance, "counter"); - int id_gauge = fieldstat_register_hll(instance, "gauge", g_hll_standard->cfg.precision); - int id_histogram = fieldstat_register_hist(instance, "histogram", + int cube_id = fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3); + int id_counter = fieldstat_register_counter(instance, cube_id, "counter"); + int id_gauge = fieldstat_register_hll(instance, cube_id, "gauge", g_hll_standard->cfg.precision); + int id_histogram = fieldstat_register_histogram(instance, cube_id, "histogram", g_histogram_standard->lowest_discernible_value, g_histogram_standard->highest_trackable_value, g_histogram_standard->significant_figures); - Fieldstat_tag_list_wrapper *tags[tag_list_num]; - fill_random_tag(tags, tag_list_num); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_COMPREHENSIVE, tag_list_num, 0); + + Fieldstat_tag_list_wrapper *fields[tag_list_num]; + fill_random_tag(fields, tag_list_num); for (int j = 0; j < tag_list_num; j++) { - const struct fieldstat_tag *tag_tmp = tags[j]->get_tag(); - size_t tag_count = tags[j]->get_tag_count(); + const struct field *tag_tmp = fields[j]->get_tag(); + size_t tag_count = fields[j]->get_tag_count(); fieldstat_counter_incrby(instance, cube_id, id_counter, tag_tmp, tag_count, 1); for (size_t i = 0; i < OPER_NUM; i++){ fieldstat_hll_add(instance, cube_id, id_gauge, tag_tmp, tag_count, g_hll_standard_oper[i].c_str(), g_hll_standard_oper[i].length()); - fieldstat_hist_record(instance, cube_id, id_histogram, tag_tmp, tag_count, g_histogram_standard_oper[i]); + fieldstat_histogram_record(instance, cube_id, id_histogram, tag_tmp, tag_count, g_histogram_standard_oper[i]); } } @@ -214,10 +218,10 @@ TEST(export_test, cjson_export_with_fixed_tag_and_many_metrics_on_one_cube_of_co { cJSON *root = cJSON_GetArrayItem(root_arr, i); - // check tag - cJSON *tag = cJSON_GetObjectItem(root, "tags"); - EXPECT_NE(tag, nullptr); - test_check_if_tag_list_is_in_json(tag, tags[i]); + // check field + cJSON *field = cJSON_GetObjectItem(root, "tags"); + EXPECT_NE(field, nullptr); + test_check_if_tag_list_is_in_json(field, fields[i]); // check metrics cJSON *metrics = cJSON_GetObjectItem(root, "fields"); @@ -229,7 +233,7 @@ TEST(export_test, cjson_export_with_fixed_tag_and_many_metrics_on_one_cube_of_co } for (int i = 0; i < 3; i++) { - delete tags[i]; + delete fields[i]; } fieldstat_free(instance); @@ -247,6 +251,7 @@ TEST(export_test, cjson_export_on_one_cube_of_topk_sampling) int arr_num = cJSON_GetArraySize(root_arr); EXPECT_EQ(arr_num, TEST_TOPK_STANDARD_K); + std::unordered_set<long long> topk_tag_value_set; for (int i = 0; i < arr_num; i++) { cJSON *root = cJSON_GetArrayItem(root_arr, i); @@ -255,11 +260,12 @@ TEST(export_test, cjson_export_on_one_cube_of_topk_sampling) cJSON *tag = cJSON_GetObjectItem(root, "tags"); EXPECT_NE(tag, nullptr); - Fieldstat_tag_list_wrapper *added_tmp = new Fieldstat_tag_list_wrapper("flow id key", i); - test_check_if_tag_list_is_in_json(tag, added_tmp); - delete added_tmp; + cJSON *tag_val = cJSON_GetObjectItem(tag, "flow id key"); + EXPECT_NE(tag_val, nullptr); + EXPECT_EQ(tag_val->type, cJSON_Number); + topk_tag_value_set.insert(tag_val->valueint); - struct fieldstat_tag_list tmp_tag = {(struct fieldstat_tag *)&TEST_TAG_SHARED1, 3}; + struct field_list tmp_tag = {(struct field *)&TEST_TAG_SHARED1, 3}; Fieldstat_tag_list_wrapper shared_wrapper = Fieldstat_tag_list_wrapper(&tmp_tag); test_check_if_tag_list_is_in_json(tag, &shared_wrapper); @@ -269,11 +275,71 @@ TEST(export_test, cjson_export_on_one_cube_of_topk_sampling) const char *metric_name[TEST_METRIC_NUM] = {"topk1", "topk2"}; test_check_if_metric_topk_correct(metrics, (const char **)metric_name, test_expected_big_count, TEST_METRIC_NUM); } + + EXPECT_EQ(topk_tag_value_set.size(), TEST_TOPK_STANDARD_K); + for (int i = 0; i < TEST_TOPK_STANDARD_K; i++) { // the top k flow id is [0, TEST_TOPK_STANDARD_K - 1] + EXPECT_NE(topk_tag_value_set.find(i), topk_tag_value_set.end()); + } + cJSON_Delete(root_arr); fieldstat_free(instance); } +extern "C" { + double fieldstat_hll_base64_to_count(char *buf); +} +TEST(export_test, cjson_export_on_one_cube_of_spreadsketch_sampling) { + int K = 10; + struct fieldstat *instance = fieldstat_new(); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_hll(instance, cube_id, "metric", 7); + int metric_count = fieldstat_register_counter(instance, cube_id, "oper cnt"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, K, 0); + + SpreadSketchZipfGenerator flow_generator(1.0, K * 10); + + for (int i = 0; i < 100000; i++) { + Flow flow = flow_generator.next(); + Fieldstat_tag_list_wrapper dimension = Fieldstat_tag_list_wrapper("bigger fanouts", flow.src_ip.c_str()); + Fieldstat_tag_list_wrapper item = Fieldstat_tag_list_wrapper("dummy", flow.dst_ip.c_str()); + fieldstat_hll_add_field(instance, cube_id, metric_id, dimension.get_tag(), dimension.get_tag_count(), item.get_tag(), item.get_tag_count()); + fieldstat_counter_incrby(instance, cube_id, metric_count, dimension.get_tag(), dimension.get_tag_count(), 1); + } + for (int i = 0; i < 100; i++) { + Flow flow = flow_generator.next(); + Fieldstat_tag_list_wrapper dimension = Fieldstat_tag_list_wrapper("smaller fanouts", flow.src_ip.c_str()); + Fieldstat_tag_list_wrapper item = Fieldstat_tag_list_wrapper("dummy", flow.dst_ip.c_str()); + fieldstat_hll_add_field(instance, cube_id, metric_id, dimension.get_tag(), dimension.get_tag_count(), item.get_tag(), item.get_tag_count()); + fieldstat_counter_incrby(instance, cube_id, metric_count, dimension.get_tag(), dimension.get_tag_count(), 1); + } + + cJSON *root_arr = test_exporter_extract_results(instance); + int arr_num = cJSON_GetArraySize(root_arr); + EXPECT_EQ(arr_num, K); + + for (int i = 0; i < arr_num; i++) { + cJSON *root = cJSON_GetArrayItem(root_arr, i); + + // check tag + cJSON *tag = cJSON_GetObjectItem(root, "tags"); + EXPECT_NE(tag, nullptr); + cJSON *tag_val = cJSON_GetObjectItem(tag, "bigger fanouts"); + EXPECT_NE(tag_val, nullptr); + + // check metrics + cJSON *metrics = cJSON_GetObjectItem(root, "fields"); + EXPECT_NE(metrics, nullptr); + cJSON *metric = cJSON_GetObjectItem(metrics, "metric"); + cJSON *metric_count = cJSON_GetObjectItem(metrics, "oper cnt"); + double hll_value = fieldstat_hll_base64_to_count(metric->valuestring); + EXPECT_NEAR(hll_value, metric_count->valueint, metric_count->valueint * 0.2); + } + + cJSON_Delete(root_arr); + fieldstat_free(instance); +} + TEST(export_test, empty_fieldstat_export_null) { struct fieldstat *instance = fieldstat_new(); cJSON *root_arr = test_exporter_extract_results_with_standard_global(instance); @@ -281,17 +347,25 @@ TEST(export_test, empty_fieldstat_export_null) { fieldstat_free(instance); } +int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k, int primary_metric_id=0) +{ + assert(mode == SAMPLING_MODE_COMPREHENSIVE); + int ret = fieldstat_cube_create(instance, dimensions, n_dimensions); + fieldstat_cube_set_sampling(instance, ret, mode, k, primary_metric_id); + return ret; +} + TEST(export_test, only_registered_but_not_added_export_null_with_global_tag) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3); - fieldstat_register_counter(instance, "counter"); - fieldstat_register_hll(instance, "gauge", g_hll_standard->cfg.precision); - fieldstat_register_hist(instance, "histogram", + int cube_id = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3); + fieldstat_register_counter(instance, cube_id, "counter"); + fieldstat_register_hll(instance, cube_id, "gauge", g_hll_standard->cfg.precision); + fieldstat_register_histogram(instance, cube_id, "histogram", g_histogram_standard->lowest_discernible_value, g_histogram_standard->highest_trackable_value, g_histogram_standard->significant_figures); - // add global tag + // add global field cJSON *root_arr = test_exporter_extract_results_with_standard_global(instance); EXPECT_EQ(root_arr, nullptr); @@ -301,20 +375,20 @@ TEST(export_test, only_registered_but_not_added_export_null_with_global_tag) TEST(export_test, skip_two_empty_cube_and_export_last_one_with_global_tag) { struct fieldstat *instance = fieldstat_new(); - (void)fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3); - (void)fieldstat_create_cube(instance, TEST_TAG_SHARED2, 3, SAMPLING_MODE_COMPREHENSIVE, 3); - int cube_id_3 = fieldstat_create_cube(instance, TEST_TAG_SHARED3, 1, SAMPLING_MODE_COMPREHENSIVE, 3); - (void)fieldstat_register_counter(instance, "counter"); - (void)fieldstat_register_hll(instance, "gauge", g_hll_standard->cfg.precision); - int id_histogram = fieldstat_register_hist(instance, "histogram", + (void)test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3); + (void)test_fieldstat_cube_create(instance, TEST_TAG_SHARED2, 3, SAMPLING_MODE_COMPREHENSIVE, 3); + int cube_id_3 = test_fieldstat_cube_create(instance, TEST_TAG_SHARED3, 1, SAMPLING_MODE_COMPREHENSIVE, 3); + (void)fieldstat_register_counter(instance, cube_id_3, "counter"); + (void)fieldstat_register_hll(instance, cube_id_3, "gauge", g_hll_standard->cfg.precision); + int id_histogram = fieldstat_register_histogram(instance, cube_id_3, "histogram", g_histogram_standard->lowest_discernible_value, g_histogram_standard->highest_trackable_value, g_histogram_standard->significant_figures); const int tag_num = 1; - Fieldstat_tag_list_wrapper *tags[tag_num]; - fill_random_tag(tags, tag_num); - const Fieldstat_tag_list_wrapper *the_tag = tags[0]; + Fieldstat_tag_list_wrapper *fields[tag_num]; + fill_random_tag(fields, tag_num); + const Fieldstat_tag_list_wrapper *the_tag = fields[0]; for (size_t i = 0; i < OPER_NUM; i++){ - fieldstat_hist_record(instance, cube_id_3, id_histogram, the_tag->get_tag(), the_tag->get_tag_count(), g_histogram_standard_oper[i]); + fieldstat_histogram_record(instance, cube_id_3, id_histogram, the_tag->get_tag(), the_tag->get_tag_count(), g_histogram_standard_oper[i]); } // export test @@ -328,7 +402,7 @@ TEST(export_test, skip_two_empty_cube_and_export_last_one_with_global_tag) // check tag cJSON *tag = cJSON_GetObjectItem(root, "tags"); - test_check_if_tag_list_is_in_json(tag, tags[0]); + test_check_if_tag_list_is_in_json(tag, fields[0]); test_check_if_global_tag_is_in_json(tag); cJSON *metrics = cJSON_GetObjectItem(root, "fields"); @@ -341,7 +415,7 @@ TEST(export_test, skip_two_empty_cube_and_export_last_one_with_global_tag) FAIL(); } - delete tags[0]; + delete fields[0]; fieldstat_free(instance); cJSON_Delete(root_arr); @@ -349,18 +423,18 @@ TEST(export_test, skip_two_empty_cube_and_export_last_one_with_global_tag) TEST(export_test, skip_empty_metrics_given_cube_deleted) { struct fieldstat *instance = fieldstat_new(); - int cube_id_del = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3); - int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED2, 3, SAMPLING_MODE_COMPREHENSIVE, 3); - fieldstat_destroy_cube(instance, cube_id_del); - (void)fieldstat_register_counter(instance, "counter"); - (void)fieldstat_register_counter(instance, "counter2"); - int metric_id = fieldstat_register_counter(instance, "counter3"); - (void)fieldstat_register_counter(instance, "counter4"); + int cube_id_del = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3); + int cube_id = test_fieldstat_cube_create(instance, TEST_TAG_SHARED2, 3, SAMPLING_MODE_COMPREHENSIVE, 3); + fieldstat_cube_destroy(instance, cube_id_del); + (void)fieldstat_register_counter(instance, cube_id, "counter"); + (void)fieldstat_register_counter(instance, cube_id, "counter2"); + int metric_id = fieldstat_register_counter(instance, cube_id, "counter3"); + (void)fieldstat_register_counter(instance, cube_id, "counter4"); const int tag_num = 1; - Fieldstat_tag_list_wrapper *tags[tag_num]; - fill_random_tag(tags, tag_num); - const Fieldstat_tag_list_wrapper *the_tag = tags[0]; + Fieldstat_tag_list_wrapper *fields[tag_num]; + fill_random_tag(fields, tag_num); + const Fieldstat_tag_list_wrapper *the_tag = fields[0]; fieldstat_counter_incrby(instance, cube_id, metric_id, the_tag->get_tag(), the_tag->get_tag_count(), 1234); @@ -371,13 +445,13 @@ TEST(export_test, skip_empty_metrics_given_cube_deleted) { for (int i = 0; i < arr_num; i++) { cJSON *root = cJSON_GetArrayItem(root_arr, i); - // check tag - cJSON *tag = cJSON_GetObjectItem(root, "tags"); - test_check_if_tag_list_is_in_json(tag, tags[0]); - test_check_if_global_tag_is_in_json(tag); - struct fieldstat_tag_list tmp_tag = {(struct fieldstat_tag *)&TEST_TAG_SHARED2, 3}; + // check field + cJSON *field = cJSON_GetObjectItem(root, "tags"); + test_check_if_tag_list_is_in_json(field, fields[0]); + test_check_if_global_tag_is_in_json(field); + struct field_list tmp_tag = {(struct field *)&TEST_TAG_SHARED2, 3}; Fieldstat_tag_list_wrapper shared_wrapper = Fieldstat_tag_list_wrapper(&tmp_tag); - test_check_if_tag_list_is_in_json(tag, &shared_wrapper); + test_check_if_tag_list_is_in_json(field, &shared_wrapper); cJSON *metrics = cJSON_GetObjectItem(root, "fields"); EXPECT_NE(metrics, nullptr); @@ -390,7 +464,7 @@ TEST(export_test, skip_empty_metrics_given_cube_deleted) { FAIL(); } - delete tags[0]; + delete fields[0]; fieldstat_free(instance); cJSON_Delete(root_arr); } @@ -398,9 +472,9 @@ TEST(export_test, skip_empty_metrics_given_cube_deleted) { TEST(export_test, enable_delta_and_export_twice_without_new_metric) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10); - int id_counter = fieldstat_register_counter(instance, "counter"); - fieldstat_counter_incrby(instance, cube_id, id_counter, &TEST_TAG_INT, 1, 1); + int cube_id = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10); + int id_counter = fieldstat_register_counter(instance, cube_id, "counter"); + fieldstat_counter_incrby(instance, cube_id, id_counter, &TEST_FIELD_INT, 1, 1); // export test struct fieldstat_json_exporter *fieldstat_json_exporter = fieldstat_json_exporter_new(); @@ -467,12 +541,12 @@ TEST(export_test, enable_delta_and_export_twice_without_new_metric) TEST(export_test, enable_delta_and_export_twice_with_new_metric_and_omit_histogram) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10); - int id_counter = fieldstat_register_counter(instance, "counter"); - int id_histogram = fieldstat_register_hist(instance, "histogram", + int cube_id = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10); + int id_counter = fieldstat_register_counter(instance, cube_id, "counter"); + int id_histogram = fieldstat_register_histogram(instance, cube_id, "histogram", g_histogram_standard->lowest_discernible_value, g_histogram_standard->highest_trackable_value, g_histogram_standard->significant_figures); - fieldstat_counter_incrby(instance, cube_id, id_counter, &TEST_TAG_INT, 1, 1); - fieldstat_hist_record(instance, cube_id, id_histogram, &TEST_TAG_INT, 1, 123); + fieldstat_counter_incrby(instance, cube_id, id_counter, &TEST_FIELD_INT, 1, 1); + fieldstat_histogram_record(instance, cube_id, id_histogram, &TEST_FIELD_INT, 1, 123); // export test struct fieldstat_json_exporter *fieldstat_json_exporter = fieldstat_json_exporter_new(); @@ -507,9 +581,9 @@ TEST(export_test, enable_delta_and_export_twice_with_new_metric_and_omit_histogr struct timeval new_ts = TEST_TIMEVAL; new_ts.tv_sec += 1; - int id_counter2 = fieldstat_register_counter(instance, "counter2"); - fieldstat_counter_incrby(instance, cube_id, id_counter2, &TEST_TAG_INT, 1, 1); - fieldstat_counter_incrby(instance, cube_id, id_counter, &TEST_TAG_INT, 1, 10); + int id_counter2 = fieldstat_register_counter(instance, cube_id, "counter2"); + fieldstat_counter_incrby(instance, cube_id, id_counter2, &TEST_FIELD_INT, 1, 1); + fieldstat_counter_incrby(instance, cube_id, id_counter, &TEST_FIELD_INT, 1, 10); json_string = fieldstat_json_exporter_export(fieldstat_json_exporter, instance, &new_ts); root_arr = cJSON_Parse(json_string); @@ -549,12 +623,12 @@ TEST(export_test, enable_delta_and_export_twice_with_new_metric_and_omit_histogr TEST(export_test, enable_delta_and_export_three_times_skipping_cube_with_no_counter) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10); - int id_counter = fieldstat_register_counter(instance, "counter"); - int id_histogram = fieldstat_register_hist(instance, "histogram", + int cube_id = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10); + int id_counter = fieldstat_register_counter(instance, cube_id, "counter"); + int id_histogram = fieldstat_register_histogram(instance, cube_id, "histogram", g_histogram_standard->lowest_discernible_value, g_histogram_standard->highest_trackable_value, g_histogram_standard->significant_figures); - fieldstat_counter_incrby(instance, cube_id, id_counter, &TEST_TAG_INT, 1, 1); - fieldstat_hist_record(instance, cube_id, id_histogram, &TEST_TAG_STRING, 1, 123); + fieldstat_counter_incrby(instance, cube_id, id_counter, &TEST_FIELD_INT, 1, 1); + fieldstat_histogram_record(instance, cube_id, id_histogram, &TEST_FIELD_STRING, 1, 123); /* -------------------------- export test, 1st time ------------------------- */ struct fieldstat_json_exporter *fieldstat_json_exporter = fieldstat_json_exporter_new(); @@ -609,8 +683,8 @@ TEST(export_test, enable_delta_and_export_three_times_skipping_cube_with_no_coun /* -------------------------------- 3rd time -------------------------------- */ // add some new recs - fieldstat_counter_incrby(instance, cube_id, id_counter, &TEST_TAG_INT, 1, 10); - fieldstat_hist_record(instance, cube_id, id_histogram, &TEST_TAG_STRING, 1, 1234); + fieldstat_counter_incrby(instance, cube_id, id_counter, &TEST_FIELD_INT, 1, 10); + fieldstat_histogram_record(instance, cube_id, id_histogram, &TEST_FIELD_STRING, 1, 1234); json_string = fieldstat_json_exporter_export(fieldstat_json_exporter, instance, &TEST_TIMEVAL); root_arr = cJSON_Parse(json_string); free(json_string); @@ -636,15 +710,15 @@ TEST(export_test, enable_delta_and_export_three_times_skipping_cube_with_no_coun fieldstat_free(instance); } -void test_check_delta_for_one_json(const struct fieldstat_tag_list *expect_cell_tag, const struct fieldstat_tag_list *expect_shared_tag, long long expect_value, long long expect_delta, cJSON *root) +void test_check_delta_for_one_json(const struct field_list *expect_cell_tag, const struct field_list *expect_shared_tag, long long expect_value, long long expect_delta, cJSON *root) { - // check tag - cJSON *tag = cJSON_GetObjectItem(root, "tags"); + // check field + cJSON *field = cJSON_GetObjectItem(root, "tags"); Fieldstat_tag_list_wrapper cell_wrapper = Fieldstat_tag_list_wrapper(expect_cell_tag); - test_check_if_tag_list_is_in_json(tag, &cell_wrapper); + test_check_if_tag_list_is_in_json(field, &cell_wrapper); Fieldstat_tag_list_wrapper shared_wrapper = Fieldstat_tag_list_wrapper(expect_shared_tag); - test_check_if_tag_list_is_in_json(tag, &shared_wrapper); - test_check_if_global_tag_is_in_json(tag); + test_check_if_tag_list_is_in_json(field, &shared_wrapper); + test_check_if_global_tag_is_in_json(field); // check metrics cJSON *metrics = cJSON_GetObjectItem(root, "fields"); @@ -658,14 +732,15 @@ void test_check_delta_for_one_json(const struct fieldstat_tag_list *expect_cell_ TEST(export_test, enable_delta_and_export_instance_with_many_cells_with_global_tags) { struct fieldstat *instance = fieldstat_new(); - int cube_id1 = fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10); - int id_counter1 = fieldstat_register_counter(instance, "counter"); - fieldstat_counter_incrby(instance, cube_id1, id_counter1, &TEST_TAG_INT, 1, 11); - fieldstat_counter_incrby(instance, cube_id1, id_counter1, &TEST_TAG_STRING, 1, 12); + int cube_id1 = test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 10); + int id_counter1 = fieldstat_register_counter(instance, cube_id1, "counter"); + fieldstat_counter_incrby(instance, cube_id1, id_counter1, &TEST_FIELD_INT, 1, 11); + fieldstat_counter_incrby(instance, cube_id1, id_counter1, &TEST_FIELD_STRING, 1, 12); - int cube_id2 = fieldstat_create_cube(instance, TEST_TAG_SHARED3, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - fieldstat_counter_incrby(instance, cube_id2, id_counter1, &TEST_TAG_INT, 1, 21); - fieldstat_counter_incrby(instance, cube_id2, id_counter1, &TEST_TAG_DOUBLE, 1, 22); + int cube_id2 = test_fieldstat_cube_create(instance, TEST_TAG_SHARED3, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int id_counter2 = fieldstat_register_counter(instance, cube_id2, "counter"); + fieldstat_counter_incrby(instance, cube_id2, id_counter2, &TEST_FIELD_INT, 1, 21); + fieldstat_counter_incrby(instance, cube_id2, id_counter2, &TEST_FIELD_DOUBLE, 1, 22); // export test struct fieldstat_json_exporter *fieldstat_json_exporter = fieldstat_json_exporter_new(); @@ -681,28 +756,28 @@ TEST(export_test, enable_delta_and_export_instance_with_many_cells_with_global_t /* ------------------------------ cube 0 cell 0 ----------------------------- */ cJSON *root = cJSON_GetArrayItem(root_arr, 0); - struct fieldstat_tag_list tmp_tag_cell = {(struct fieldstat_tag *)&TEST_TAG_INT, 1}; - struct fieldstat_tag_list tmp_tag_shared = {(struct fieldstat_tag *)&TEST_TAG_SHARED1, 3}; + struct field_list tmp_tag_cell = {(struct field *)&TEST_FIELD_INT, 1}; + struct field_list tmp_tag_shared = {(struct field *)&TEST_TAG_SHARED1, 3}; test_check_delta_for_one_json(&tmp_tag_cell, &tmp_tag_shared, 11, 11, root); /* ------------------------------ cube 0 cell 1 ----------------------------- */ root = cJSON_GetArrayItem(root_arr, 1); - tmp_tag_cell = (struct fieldstat_tag_list){(struct fieldstat_tag *)&TEST_TAG_STRING, 1}; + tmp_tag_cell = (struct field_list){(struct field *)&TEST_FIELD_STRING, 1}; test_check_delta_for_one_json(&tmp_tag_cell, &tmp_tag_shared, 12, 12, root); /* ------------------------------ cube 1 cell 0 ----------------------------- */ root = cJSON_GetArrayItem(root_arr, 2); - tmp_tag_cell = (struct fieldstat_tag_list){(struct fieldstat_tag *)&TEST_TAG_INT, 1}; - tmp_tag_shared = (struct fieldstat_tag_list){(struct fieldstat_tag *)&TEST_TAG_SHARED3, 1}; + tmp_tag_cell = (struct field_list){(struct field *)&TEST_FIELD_INT, 1}; + tmp_tag_shared = (struct field_list){(struct field *)&TEST_TAG_SHARED3, 1}; test_check_delta_for_one_json(&tmp_tag_cell, &tmp_tag_shared, 21, 21, root); /* ------------------------------ cube 1 cell 1 ----------------------------- */ root = cJSON_GetArrayItem(root_arr, 3); - tmp_tag_cell = (struct fieldstat_tag_list){(struct fieldstat_tag *)&TEST_TAG_DOUBLE, 1}; + tmp_tag_cell = (struct field_list){(struct field *)&TEST_FIELD_DOUBLE, 1}; test_check_delta_for_one_json(&tmp_tag_cell, &tmp_tag_shared, 22, 22, root); // new turn - fieldstat_counter_incrby(instance, cube_id1, id_counter1, &TEST_TAG_INT, 1, 100); - fieldstat_counter_incrby(instance, cube_id1, id_counter1, &TEST_TAG_STRING, 1, 200); - fieldstat_counter_incrby(instance, cube_id2, id_counter1, &TEST_TAG_INT, 1, 300); - fieldstat_counter_incrby(instance, cube_id2, id_counter1, &TEST_TAG_DOUBLE, 1, 400); + fieldstat_counter_incrby(instance, cube_id1, id_counter1, &TEST_FIELD_INT, 1, 100); + fieldstat_counter_incrby(instance, cube_id1, id_counter1, &TEST_FIELD_STRING, 1, 200); + fieldstat_counter_incrby(instance, cube_id2, id_counter1, &TEST_FIELD_INT, 1, 300); + fieldstat_counter_incrby(instance, cube_id2, id_counter1, &TEST_FIELD_DOUBLE, 1, 400); cJSON_Delete(root_arr); @@ -717,21 +792,21 @@ TEST(export_test, enable_delta_and_export_instance_with_many_cells_with_global_t /* ------------------------------ cube 0 cell 0 ----------------------------- */ root = cJSON_GetArrayItem(root_arr, 0); - tmp_tag_cell = (struct fieldstat_tag_list){(struct fieldstat_tag *)&TEST_TAG_INT, 1}; - tmp_tag_shared = (struct fieldstat_tag_list){(struct fieldstat_tag *)&TEST_TAG_SHARED1, 3}; + tmp_tag_cell = (struct field_list){(struct field *)&TEST_FIELD_INT, 1}; + tmp_tag_shared = (struct field_list){(struct field *)&TEST_TAG_SHARED1, 3}; test_check_delta_for_one_json(&tmp_tag_cell, &tmp_tag_shared, 111, 100, root); /* ------------------------------ cube 0 cell 1 ----------------------------- */ root = cJSON_GetArrayItem(root_arr, 1); - tmp_tag_cell = (struct fieldstat_tag_list){(struct fieldstat_tag *)&TEST_TAG_STRING, 1}; + tmp_tag_cell = (struct field_list){(struct field *)&TEST_FIELD_STRING, 1}; test_check_delta_for_one_json(&tmp_tag_cell, &tmp_tag_shared, 212, 200, root); /* ------------------------------ cube 1 cell 0 ----------------------------- */ root = cJSON_GetArrayItem(root_arr, 2); - tmp_tag_cell = (struct fieldstat_tag_list){(struct fieldstat_tag *)&TEST_TAG_INT, 1}; - tmp_tag_shared = (struct fieldstat_tag_list){(struct fieldstat_tag *)&TEST_TAG_SHARED3, 1}; + tmp_tag_cell = (struct field_list){(struct field *)&TEST_FIELD_INT, 1}; + tmp_tag_shared = (struct field_list){(struct field *)&TEST_TAG_SHARED3, 1}; test_check_delta_for_one_json(&tmp_tag_cell, &tmp_tag_shared, 321, 300, root); /* ------------------------------ cube 1 cell 1 ----------------------------- */ root = cJSON_GetArrayItem(root_arr, 3); - tmp_tag_cell = (struct fieldstat_tag_list){(struct fieldstat_tag *)&TEST_TAG_DOUBLE, 1}; + tmp_tag_cell = (struct field_list){(struct field *)&TEST_FIELD_DOUBLE, 1}; test_check_delta_for_one_json(&tmp_tag_cell, &tmp_tag_shared, 422, 400, root); cJSON_Delete(root_arr); @@ -743,9 +818,9 @@ TEST(export_test, enable_delta_and_export_instance_with_many_cells_with_global_t void test_reset_one_round(std::function<void(struct fieldstat *, struct fieldstat_json_exporter *)> trigger_reset) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 1); - int id_counter = fieldstat_register_counter(instance, "counter"); - fieldstat_counter_incrby(instance, cube_id, id_counter, &TEST_TAG_INT, 1, 11); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 1); + int id_counter = fieldstat_register_counter(instance, cube_id, "counter"); + fieldstat_counter_incrby(instance, cube_id, id_counter, &TEST_FIELD_INT, 1, 11); // export test struct fieldstat_json_exporter *fieldstat_json_exporter = fieldstat_json_exporter_new(); @@ -767,9 +842,10 @@ void test_reset_one_round(std::function<void(struct fieldstat *, struct fieldsta trigger_reset(instance, fieldstat_json_exporter); - fieldstat_counter_set(instance, cube_id, id_counter, &TEST_TAG_INT, 1, 123); + fieldstat_counter_set(instance, cube_id, id_counter, &TEST_FIELD_INT, 1, 123); json_string = fieldstat_json_exporter_export(fieldstat_json_exporter, instance, &TEST_TIMEVAL); + printf("test_reset_one_round :\n%s\n", json_string); root_arr = cJSON_Parse(json_string); free(json_string); @@ -804,18 +880,11 @@ TEST(export_test, enable_delta_and_reset_on_change_exporter_tag) { test_reset_one_round(trigger); } -TEST(export_test, enable_delta_and_reset_on_reset_instance) { - auto trigger = [](struct fieldstat *instance, struct fieldstat_json_exporter *fieldstat_json_exporter) { - fieldstat_reset(instance); - }; - - test_reset_one_round(trigger); -} - TEST(export_test, enable_delta_and_reset_on_delete_cube) { auto trigger = [](struct fieldstat *instance, struct fieldstat_json_exporter *fieldstat_json_exporter) { - fieldstat_destroy_cube(instance, 0); - fieldstat_create_cube(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3); + fieldstat_cube_destroy(instance, 0); + test_fieldstat_cube_create(instance, TEST_TAG_SHARED1, 3, SAMPLING_MODE_COMPREHENSIVE, 3); + fieldstat_register_counter(instance, 0, "counter"); }; test_reset_one_round(trigger); @@ -824,15 +893,15 @@ TEST(export_test, enable_delta_and_reset_on_delete_cube) { TEST(export_test, delta_with_two_instance_same_config) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); - int id_counter = fieldstat_register_counter(instance, "counter"); - fieldstat_counter_incrby(instance, 0, id_counter, &TEST_TAG_INT, 1, 123); - int id_hist = fieldstat_register_hist(instance, "histogram", 1, 1000, 3); - fieldstat_hist_record(instance, 0, id_hist, &TEST_TAG_INT, 1, 5); + int cube_id = test_fieldstat_cube_create(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); + int id_counter = fieldstat_register_counter(instance, cube_id, "counter"); + fieldstat_counter_incrby(instance, 0, id_counter, &TEST_FIELD_INT, 1, 123); + int id_hist = fieldstat_register_histogram(instance, cube_id, "histogram", 1, 1000, 3); + fieldstat_histogram_record(instance, cube_id, id_hist, &TEST_FIELD_INT, 1, 5); struct fieldstat *acc = fieldstat_new(); fieldstat_merge(acc, instance); - fieldstat_counter_incrby(acc, 0, id_counter, &TEST_TAG_INT, 1, 1000); + fieldstat_counter_incrby(acc, 0, id_counter, &TEST_FIELD_INT, 1, 1000); // export test struct fieldstat_json_exporter *fieldstat_json_exporter = fieldstat_json_exporter_new(); @@ -864,18 +933,17 @@ TEST(export_test, delta_with_two_instance_same_config) TEST(export_test, delta_with_two_instance_one_empty) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); - int id_counter = fieldstat_register_counter(instance, "counter"); - fieldstat_counter_incrby(instance, 0, id_counter, &TEST_TAG_INT, 1, 123); - int id_hist = fieldstat_register_hist(instance, "histogram", 1, 1000, 3); - fieldstat_hist_record(instance, 0, id_hist, &TEST_TAG_INT, 1, 5); + int cube_id = test_fieldstat_cube_create(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); + int id_counter = fieldstat_register_counter(instance, cube_id, "counter"); + fieldstat_counter_incrby(instance, 0, id_counter, &TEST_FIELD_INT, 1, 123); + int id_hist = fieldstat_register_histogram(instance, cube_id, "histogram", 1, 1000, 3); + fieldstat_histogram_record(instance, 0, id_hist, &TEST_FIELD_INT, 1, 5); struct fieldstat *delta = fieldstat_fork(instance); // export test struct fieldstat_json_exporter *fieldstat_json_exporter = fieldstat_json_exporter_new(); char *json = fieldstat_json_exporter_export_with_delta(fieldstat_json_exporter, instance, delta, &TEST_TIMEVAL, &TEST_TIMEVAL); - printf("delta_with_two_instance_one_empty :\n%s\n", json); cJSON *root_arr = cJSON_Parse(json); free(json); @@ -904,12 +972,12 @@ TEST(export_test, delta_with_two_instance_one_empty) TEST(export_test, delta_with_two_instance_different_cell) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); - int id_counter = fieldstat_register_counter(instance, "counter"); - fieldstat_counter_incrby(instance, 0, id_counter, &TEST_TAG_INT, 1, 123); + test_fieldstat_cube_create(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); + int id_counter = fieldstat_register_counter(instance, 0, "counter"); + fieldstat_counter_incrby(instance, 0, id_counter, &TEST_FIELD_INT, 1, 123); struct fieldstat *delta = fieldstat_fork(instance); - fieldstat_counter_incrby(delta, 0, id_counter, &TEST_TAG_DOUBLE, 1, 1); + fieldstat_counter_incrby(delta, 0, id_counter, &TEST_FIELD_DOUBLE, 1, 1); fieldstat_merge(instance, delta); // export test @@ -949,27 +1017,66 @@ TEST(export_test, delta_with_two_instance_different_cell) fieldstat_free(delta); } -// TEST(export_test, sort_tag) { -// const struct fieldstat_tag tag1[3] = {TEST_TAG_INT, TEST_TAG_STRING, TEST_TAG_DOUBLE}; -// const struct fieldstat_tag tag2[3] = {TEST_TAG_STRING, TEST_TAG_DOUBLE, TEST_TAG_INT}; +TEST(export_test, export_flat_null_with_only_global_tag) { + struct fieldstat *instance = fieldstat_new(); + struct fieldstat_json_exporter *fieldstat_json_exporter = fieldstat_json_exporter_new(); + fieldstat_json_exporter_set_global_tag(fieldstat_json_exporter, TEST_TAG_GLOBAL, 3); + + char **out_array = NULL; + size_t out_array_len = 0; + fieldstat_json_exporter_export_flat_array(fieldstat_json_exporter, instance, &TEST_TIMEVAL, &out_array, &out_array_len); + EXPECT_EQ(out_array_len, 0); + EXPECT_EQ(out_array, nullptr); + + fieldstat_json_exporter_free(fieldstat_json_exporter); + free(out_array); + fieldstat_free(instance); +} + +TEST(export_test, export_flat_many_tags_many_cell) { + struct fieldstat *instance = fieldstat_new(); + int cube_id = test_fieldstat_cube_create(instance, &TEST_FIELD_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 3); + int metric_id = fieldstat_register_counter(instance, cube_id, "counter metric"); -// struct fieldstat *instance = fieldstat_new(); -// fieldstat_create_cube(instance, NULL, 0, SAMPLING_MODE_COMPREHENSIVE, 0); -// int id_counter = fieldstat_register_counter(instance, "counter"); -// fieldstat_counter_incrby(instance, 0, id_counter, tag1, 3, 1); -// fieldstat_counter_incrby(instance, 0, id_counter, tag2, 3, 10); -// // export test -// cJSON *root_arr = test_exporter_extract_results(instance); -// EXPECT_EQ(cJSON_GetArraySize(root_arr), 1); -// cJSON *root = cJSON_GetArrayItem(root_arr, 0); -// cJSON *metrics = cJSON_GetObjectItem(root, "fields"); -// cJSON *counter = cJSON_GetObjectItem(metrics, "counter"); -// EXPECT_EQ(counter->valueint, 11); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, 123); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_DOUBLE, 1, 321); -// cJSON_Delete(root_arr); -// fieldstat_free(instance); -// } + struct fieldstat_json_exporter *fieldstat_json_exporter = fieldstat_json_exporter_new(); + fieldstat_json_exporter_set_global_tag(fieldstat_json_exporter, TEST_TAG_GLOBAL, 3); + + char **out_array = NULL; + size_t out_array_len = 0; + fieldstat_json_exporter_export_flat_array(fieldstat_json_exporter, instance, &TEST_TIMEVAL, &out_array, &out_array_len); + EXPECT_EQ(out_array_len, 2); + + cJSON *root = cJSON_Parse(out_array[0]); + Fieldstat_tag_list_wrapper cell_dimension = Fieldstat_tag_list_wrapper(&TEST_FIELD_LIST_INT); + Fieldstat_tag_list_wrapper cube_dimension = Fieldstat_tag_list_wrapper(&TEST_FIELD_LIST_STRING); + Fieldstat_tag_list_wrapper cell_dimension2 = Fieldstat_tag_list_wrapper(&TEST_FIELD_LIST_DOUBLE); + + test_check_if_tag_list_is_in_json(root, &cell_dimension); + test_check_if_global_tag_is_in_json(root); + test_check_if_tag_list_is_in_json(root, &cube_dimension); + cJSON *metrics = cJSON_GetObjectItem(root, "counter metric"); + EXPECT_EQ(metrics->valueint, 123); + + cJSON_Delete(root); + root = cJSON_Parse(out_array[1]); + test_check_if_tag_list_is_in_json(root, &cell_dimension2); + test_check_if_global_tag_is_in_json(root); + test_check_if_tag_list_is_in_json(root, &cube_dimension); + metrics = cJSON_GetObjectItem(root, "counter metric"); + EXPECT_EQ(metrics->valueint, 321); + + cJSON_Delete(root); + fieldstat_free(instance); + fieldstat_json_exporter_free(fieldstat_json_exporter); + for (size_t i = 0; i < out_array_len; i++) { + free(out_array[i]); + } + free(out_array); +} extern "C" { extern int add_object_to_json_array_start(char *buf, int buf_len); @@ -1056,11 +1163,11 @@ TEST(export_unit_test, json_writer_length_is_on_margin_4096_string) void init_hll_standard_oper() { - g_hll_standard = ST_hyperloglog_new(12); + g_hll_standard = hyperloglog_new(12); for (size_t i = 0; i < OPER_NUM; i++) { std::string added_tmp = std::to_string(i); g_hll_standard_oper[i] = added_tmp; - ST_hyperloglog_add(g_hll_standard, added_tmp.c_str(), added_tmp.size()); + hyperloglog_add(g_hll_standard, added_tmp.c_str(), added_tmp.size()); } } @@ -1079,10 +1186,12 @@ int main(int argc, char *argv[]) { init_hll_standard_oper(); init_histogram_standard_oper(); + testing::InitGoogleTest(&argc, argv); + // testing::GTEST_FLAG(filter) = "*flat*"; int ret = RUN_ALL_TESTS(); - ST_hyperloglog_free(g_hll_standard); + hyperloglog_free(g_hll_standard); hdr_close(g_histogram_standard); return ret; } diff --git a/test/test_fieldstat_exporter.py b/test/test_fieldstat_exporter.py index aaafb16..4f82f1a 100644 --- a/test/test_fieldstat_exporter.py +++ b/test/test_fieldstat_exporter.py @@ -17,6 +17,7 @@ from prettytable import PrettyTable,NONE,HEADER import os current_path = os.path.dirname(os.path.abspath(__file__)) sys.path.append(current_path + '/../../src/exporter') +sys.path.append('/home/chenzizhan/heavykeeperczz/FieldStat/src/exporter') from fieldstat_exporter import FieldstatAPI @@ -153,7 +154,7 @@ class TestPrometheusExporter(unittest.TestCase): metric = self.prom._PrometheusExporter__build_type_hll(name, tags, value) - self.assertEqual(metric, "tsg_master_log{policy_id=\"1\",app_name=\"-\"} 62.61\n") + self.assertEqual(metric, "tsg_master_log{policy_id=\"1\",app_name=\"-\"} 93.61\n") def test__build_metrics(self): @@ -186,7 +187,7 @@ class TestPrometheusExporter(unittest.TestCase): "timestamp_ms_delta": 100010 } metrics = self.prom._PrometheusExporter__build_metrics(hll_dict) - self.assertEqual(metrics, "external_ip{rule_id=\"1\",app_name=\"-\"} 62.61\n") + self.assertEqual(metrics, "external_ip{rule_id=\"1\",app_name=\"-\"} 93.61\n") hist_dict = {"name": "-", "tags": { diff --git a/test/test_fuzz_test.cpp b/test/test_fuzz_test.cpp index 2083b89..46f60e4 100644 --- a/test/test_fuzz_test.cpp +++ b/test/test_fuzz_test.cpp @@ -10,7 +10,7 @@ using namespace std; -void fill_random_tag_of_length_1_to_3(Fieldstat_tag_list_wrapper *tags[], int tag_list_num) +void fill_random_tag_of_length_1_to_3(Fieldstat_tag_list_wrapper *fields[], int tag_list_num) { std::uniform_int_distribution<int> dist(1,100); std::mt19937 rng(); @@ -19,39 +19,39 @@ void fill_random_tag_of_length_1_to_3(Fieldstat_tag_list_wrapper *tags[], int ta { Fieldstat_tag_list_wrapper *tmp = new Fieldstat_tag_list_wrapper(dist, rand() % 3 + 1); tmp->sort_tag_list(); - tags[i] = tmp; + fields[i] = tmp; } } -void fill_with_elephant_flows(Fieldstat_tag_list_wrapper *tags[], int tag_list_num) +void fill_with_elephant_flows(Fieldstat_tag_list_wrapper *fields[], int tag_list_num) { for (int i = 0; i < tag_list_num; i++) { Fieldstat_tag_list_wrapper *tmp; - int rand_ret = rand() % 3; - if (rand_ret == 0) { - tmp = new Fieldstat_tag_list_wrapper("mouse", rand() % 1000); - } else if (rand_ret == 1) { - tmp = new Fieldstat_tag_list_wrapper("elephant", rand() % 200); - } else { + int rand_ret = rand() % 10; + if (rand_ret < 5) { tmp = new Fieldstat_tag_list_wrapper("elephant", rand() % 50); // most hit + } else if (rand_ret == 6 || rand_ret == 7) { + tmp = new Fieldstat_tag_list_wrapper("mid", rand() % 200); + } else { + tmp = new Fieldstat_tag_list_wrapper("mouse", rand() % 10000); } - tags[i] = tmp; + fields[i] = tmp; } } -long long fuzz_fieldstat_counter_get(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tag_list) +long long fuzz_fieldstat_counter_get(const struct fieldstat *instance, int cube_id, int metric_id, const struct field_list *tag_list) { long long value = 0; - int ret = fieldstat_counter_get(instance, cube_id, metric_id, tag_list, &value); + int ret = fieldstat_counter_get(instance, cube_id, tag_list, metric_id, &value); EXPECT_EQ(ret, 0); return value; } -double fuzz_fieldstat_hll_get(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tag_list) +double fuzz_fieldstat_hll_get(const struct fieldstat *instance, int cube_id, int metric_id, const struct field_list *tag_list) { double value = 0; - int ret = fieldstat_hll_get(instance, cube_id, metric_id, tag_list, &value); + int ret = fieldstat_hll_get(instance, cube_id, tag_list, metric_id, &value); EXPECT_EQ(ret, 0); return value; } @@ -77,13 +77,15 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_ // init cube for (int i = 0; i < CUBE_NUM; i++) { shared_tags[i] = new Fieldstat_tag_list_wrapper("shared_tag", i); - int cube_id = fieldstat_create_cube(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count(), SAMPLING_MODE_COMPREHENSIVE, CELL_MAX); + int cube_id = fieldstat_cube_create(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count()); EXPECT_EQ(cube_id, i); + + fieldstat_register_counter(master, cube_id, metric_name[METRIC_ID_COUNTER]); + fieldstat_register_hll(master, cube_id, metric_name[METRIC_ID_HLL], 6); + fieldstat_cube_set_sampling(master, cube_id, SAMPLING_MODE_COMPREHENSIVE, CELL_MAX, 0); } - // init metric - fieldstat_register_counter(master, metric_name[METRIC_ID_COUNTER]); - fieldstat_register_hll(master, metric_name[METRIC_ID_HLL], 6); - // all the possible tags + + // all the possible fields Fieldstat_tag_list_wrapper *tag_list_wrapper[FLOW_NUM]; fill_random_tag_of_length_1_to_3(tag_list_wrapper, FLOW_NUM); //all the possible operations @@ -119,8 +121,12 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_ Fieldstat_tag_list_wrapper *new_tag = new Fieldstat_tag_list_wrapper("shared_tag", next_shared_tag_value++); delete shared_tags[cube_id_to_change]; shared_tags[cube_id_to_change] = new_tag; - fieldstat_destroy_cube(master, cube_id_to_change); - int cube_id_new = fieldstat_create_cube(master, new_tag->get_tag(), new_tag->get_tag_count(), SAMPLING_MODE_COMPREHENSIVE, CELL_MAX); + fieldstat_cube_destroy(master, cube_id_to_change); + int cube_id_new = fieldstat_cube_create(master, new_tag->get_tag(), new_tag->get_tag_count()); + fieldstat_register_counter(master, cube_id_new, metric_name[METRIC_ID_COUNTER]); + fieldstat_register_hll(master, cube_id_new, metric_name[METRIC_ID_HLL], 6); + fieldstat_cube_set_sampling(master, cube_id_new, SAMPLING_MODE_COMPREHENSIVE, CELL_MAX, 0); + EXPECT_EQ(cube_id_new, cube_id_to_change); // should new the cube in the hole leaved by the destroyed cube // calibrate for (int j = 0; j < INSTANCE_NUM; j++) { @@ -133,19 +139,19 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_ } } struct fieldstat *instance = replica[rand() % INSTANCE_NUM]; // the flow randomly goes to one of the instance - const Fieldstat_tag_list_wrapper * tag = tag_list_wrapper[rand() % FLOW_NUM]; + const Fieldstat_tag_list_wrapper * field = tag_list_wrapper[rand() % FLOW_NUM]; int cube_id = rand() % CUBE_NUM; const Fieldstat_tag_list_wrapper *shared_tag = shared_tags[cube_id]; - int ret_add = fieldstat_counter_incrby(instance, cube_id, METRIC_ID_COUNTER, tag->get_tag(), tag->get_tag_count(), rand_nums[i]); + int ret_add = fieldstat_counter_incrby(instance, cube_id, METRIC_ID_COUNTER, field->get_tag(), field->get_tag_count(), rand_nums[i]); if (ret_add == FS_ERR_TOO_MANY_CELLS) { continue; } EXPECT_EQ(ret_add, FS_OK); string *val = rand_strs[i]; - ret_add = fieldstat_hll_add(instance, cube_id, METRIC_ID_HLL, tag->get_tag(), tag->get_tag_count(), val->c_str(), val->size()); + ret_add = fieldstat_hll_add(instance, cube_id, METRIC_ID_HLL, field->get_tag(), field->get_tag_count(), val->c_str(), val->size()); EXPECT_EQ(ret_add, FS_OK); - string cell_key = shared_tag->to_string() + tag->to_string(); + string cell_key = shared_tag->to_string() + field->to_string(); comp_count[cell_key] += rand_nums[i]; comp_hll[cell_key].insert(*val); } @@ -163,23 +169,22 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_ delete shared_tags[i]; } + long long dummy_ll; + double dummy_d; int *cube_ids; int cube_num; struct fieldstat *instance_in_focus = dest; fieldstat_get_cubes(instance_in_focus, &cube_ids, &cube_num); for (int i = 0; i < cube_num; i++) { - struct fieldstat_tag_list *shared_tag_out = fieldstat_get_shared_tags(instance_in_focus, cube_ids[i]); + struct field_list *shared_tag_out = fieldstat_cube_get_dimensions(instance_in_focus, cube_ids[i]); size_t cell_num0; - struct fieldstat_tag_list *tags0; - fieldstat_get_cells_used_by_metric(instance_in_focus, cube_ids[i], METRIC_ID_COUNTER, &tags0, &cell_num0); - size_t cell_num1; - struct fieldstat_tag_list *tags1; - fieldstat_get_cells_used_by_metric(instance_in_focus, cube_ids[i], METRIC_ID_HLL, &tags1, &cell_num1); + struct field_list *tags0; + fieldstat_cube_get_cells(instance_in_focus, cube_ids[i], &tags0, &cell_num0); - EXPECT_EQ(cell_num0, cell_num1); for (size_t j = 0; j < cell_num0; j++) { - EXPECT_TRUE(Fieldstat_tag_list_wrapper(&tags0[j]) == Fieldstat_tag_list_wrapper(&tags1[j])); + EXPECT_EQ(fieldstat_counter_get(instance_in_focus, cube_ids[i], &tags0[j], METRIC_ID_COUNTER, &dummy_ll), FS_OK); + EXPECT_EQ(fieldstat_hll_get(instance_in_focus, cube_ids[i], &tags0[j], METRIC_ID_HLL, &dummy_d), FS_OK); } for (size_t j = 0; j < cell_num0; j++) { @@ -189,9 +194,8 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_ EXPECT_EQ(comp_count[cell_key], fuzz_fieldstat_counter_get(instance_in_focus, cube_ids[i], 0, &tags0[j])); } - fieldstat_tag_list_arr_free(tags0, cell_num0); - fieldstat_tag_list_arr_free(tags1, cell_num1); - fieldstat_tag_list_arr_free(shared_tag_out, 1); + fieldstat_field_list_arr_free(tags0, cell_num0); + fieldstat_field_list_arr_free(shared_tag_out, 1); } free(cube_ids); @@ -219,12 +223,13 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_ // init cube for (int i = 0; i < CUBE_NUM; i++) { shared_tags[i] = new Fieldstat_tag_list_wrapper("shared_tag", i); - int cube_id = fieldstat_create_cube(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count(), SAMPLING_MODE_TOPK, CELL_MAX); + int cube_id = fieldstat_cube_create(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count()); EXPECT_EQ(cube_id, i); + fieldstat_register_counter(master, cube_id, "topk"); + fieldstat_cube_set_sampling(master, cube_id, SAMPLING_MODE_TOPK, CELL_MAX, 0); } - // init metric - fieldstat_register_counter(master, "topk"); - // all the possible tags + + // all the possible fields Fieldstat_tag_list_wrapper *tag_list_wrapper[FLOW_NUM]; fill_with_elephant_flows(tag_list_wrapper, FLOW_NUM); //all the possible operations @@ -237,7 +242,7 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_ replica[i] = fieldstat_fork(master); } // for benchmark - unordered_map<string, unordered_map<string, int>> count_map; // hte first key is shared tag, second key is tag + unordered_map<string, unordered_map<string, int>> count_map; // hte first key is shared field, second key is field clock_t start = clock(); int next_shared_tag_value = CUBE_NUM; @@ -257,8 +262,11 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_ Fieldstat_tag_list_wrapper *new_tag = new Fieldstat_tag_list_wrapper("shared_tag", next_shared_tag_value++); delete shared_tags[cube_id_to_change]; shared_tags[cube_id_to_change] = new_tag; - fieldstat_destroy_cube(master, cube_id_to_change); - int cube_id_new = fieldstat_create_cube(master, new_tag->get_tag(), new_tag->get_tag_count(), SAMPLING_MODE_TOPK, CELL_MAX); + fieldstat_cube_destroy(master, cube_id_to_change); + int cube_id_new = fieldstat_cube_create(master, new_tag->get_tag(), new_tag->get_tag_count()); + fieldstat_register_counter(master, cube_id_new, "topk"); + fieldstat_cube_set_sampling(master, cube_id_new, SAMPLING_MODE_TOPK, CELL_MAX, 0); + EXPECT_EQ(cube_id_new, cube_id_to_change); // should new the cube in the hole leaved by the destroyed cube // calibrate for (int j = 0; j < INSTANCE_NUM; j++) { @@ -271,16 +279,16 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_ } } struct fieldstat *instance = replica[rand() % INSTANCE_NUM]; // the flow randomly goes to one of the instance - const Fieldstat_tag_list_wrapper * tag = tag_list_wrapper[rand() % FLOW_NUM]; + const Fieldstat_tag_list_wrapper * field = tag_list_wrapper[rand() % FLOW_NUM]; int cube_id = rand() % CUBE_NUM; const Fieldstat_tag_list_wrapper *shared_tag = shared_tags[cube_id]; - int ret_add = fieldstat_counter_incrby(instance, cube_id, 0, tag->get_tag(), tag->get_tag_count(), rand_nums[i]); + int ret_add = fieldstat_counter_incrby(instance, cube_id, 0, field->get_tag(), field->get_tag_count(), rand_nums[i]); if (ret_add == FS_ERR_TOO_MANY_CELLS) { continue; } EXPECT_EQ(ret_add, FS_OK); - count_map[shared_tag->to_string()][tag->to_string()] += rand_nums[i]; + count_map[shared_tag->to_string()][field->to_string()] += rand_nums[i]; } clock_t end = clock(); @@ -298,25 +306,41 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_ struct fieldstat *instance_in_focus = dest; fieldstat_get_cubes(instance_in_focus, &cube_ids, &cube_num); for (int i = 0; i < cube_num; i++) { - struct fieldstat_tag_list *shared_tag_out = fieldstat_get_shared_tags(instance_in_focus, cube_ids[i]); + struct field_list *shared_tag_out = fieldstat_cube_get_dimensions(instance_in_focus, cube_ids[i]); size_t cell_num; - struct fieldstat_tag_list *tags; - fieldstat_get_cells_used_by_metric(instance_in_focus, cube_ids[i], 0, &tags, &cell_num); + struct field_list *fields; + fieldstat_cube_get_cells(instance_in_focus, cube_ids[i], &fields, &cell_num); std::vector<struct Fieldstat_tag_list_wrapper *> test_result; for (size_t j = 0; j < cell_num; j++) { - test_result.push_back(new Fieldstat_tag_list_wrapper(&tags[j])); + test_result.push_back(new Fieldstat_tag_list_wrapper(&fields[j])); } - EXPECT_GE(test_cal_topk_accuracy(test_result, count_map[Fieldstat_tag_list_wrapper(shared_tag_out).to_string()]), 0.9); + double accuracy = test_cal_topk_accuracy(test_result, count_map[Fieldstat_tag_list_wrapper(shared_tag_out).to_string()]); + EXPECT_GE(accuracy, 0.95); + // printf("topk accuracy: %lf\n", accuracy); + + // mre + double mre = 0; + for (size_t j = 0; j < cell_num; j++) { + Fieldstat_tag_list_wrapper cell_dimension = Fieldstat_tag_list_wrapper(&fields[j]); + long long value_true = count_map[Fieldstat_tag_list_wrapper(shared_tag_out).to_string()][cell_dimension.to_string()]; + long long value_est; + fieldstat_counter_get(instance_in_focus, cube_ids[i], &fields[j], 0, &value_est); + + mre += (double)(abs(value_true - value_est)) / (double)value_true; + } + mre = mre / cell_num; + // printf("topk_add_and_test_accuracy Mean ratio e: %f\n", mre); + EXPECT_LE(mre, 0.25); for (size_t j = 0; j < cell_num; j++) { delete test_result[j]; } - fieldstat_tag_list_arr_free(tags, cell_num); - fieldstat_tag_list_arr_free(shared_tag_out, 1); + fieldstat_field_list_arr_free(fields, cell_num); + fieldstat_field_list_arr_free(shared_tag_out, 1); } free(cube_ids); @@ -327,6 +351,154 @@ TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_ } } +TEST(Fuzz_test, many_instance_random_flow_unregister_calibrate_reset_fork_merge_spreadsketch) +{ + const int CUBE_NUM = 5; + const int INSTANCE_NUM = 10; + const int CELL_MAX = 10; + const int TEST_ROUND = 100000; + const int OUT_GAP = 10000; + struct fieldstat *master = fieldstat_new(); + struct fieldstat *replica[INSTANCE_NUM]; + struct fieldstat *dest = fieldstat_new(); + + Fieldstat_tag_list_wrapper *shared_tags[CUBE_NUM]; + + // init cube + for (int i = 0; i < CUBE_NUM; i++) { + shared_tags[i] = new Fieldstat_tag_list_wrapper("shared_tag", i); + int cube_id = fieldstat_cube_create(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count()); + EXPECT_EQ(cube_id, i); + fieldstat_register_hll(master, cube_id, "hll", 6); + fieldstat_cube_set_sampling(master, cube_id, SAMPLING_MODE_TOP_CARDINALITY, CELL_MAX, 0); + } + + //init instance + for (int i = 0; i < INSTANCE_NUM; i++) { + replica[i] = fieldstat_fork(master); + } + + SpreadSketchZipfGenerator generator(1.0, CELL_MAX * 10); + unordered_map<string, unordered_map<string, int>> count_map; // the first key is cube dimension, second key is cell dimension. value is the fanout(hll return value) + + clock_t start = clock(); + int next_shared_tag_value = CUBE_NUM; + + for (int i = 0; i < TEST_ROUND; i++) { + if (i != 0 && i % OUT_GAP == 0) { + // merge + for (int j = 0; j < INSTANCE_NUM; j++) { + fieldstat_merge(dest, replica[j]); + } + for (int j = 0; j < INSTANCE_NUM; j++) { + fieldstat_reset(replica[j]); + } + + // modify master and calibrate + int cube_id_to_change = rand() % CUBE_NUM; + Fieldstat_tag_list_wrapper *new_tag = new Fieldstat_tag_list_wrapper("shared_tag", next_shared_tag_value++); + delete shared_tags[cube_id_to_change]; + shared_tags[cube_id_to_change] = new_tag; + fieldstat_cube_destroy(master, cube_id_to_change); + int cube_id_new = fieldstat_cube_create(master, new_tag->get_tag(), new_tag->get_tag_count()); + fieldstat_register_hll(master, cube_id_new, "hll", 6); + EXPECT_EQ(cube_id_new, cube_id_to_change); // should new the cube in the hole leaved by the destroyed cube + fieldstat_cube_set_sampling(master, cube_id_new, SAMPLING_MODE_TOP_CARDINALITY, CELL_MAX, 0); + + // calibrate + for (int j = 0; j < INSTANCE_NUM; j++) { + fieldstat_calibrate(master, replica[j]); + } + + // let merge happens last(no add operation is missed) + if (i + OUT_GAP >= TEST_ROUND) { + break; + } + } + + // add + Flow flow = generator.next(); + + struct fieldstat *instance = replica[rand() % INSTANCE_NUM]; + const Fieldstat_tag_list_wrapper cell_dimension("src_ip", flow.src_ip.c_str()); + const Fieldstat_tag_list_wrapper item("dst_ip", flow.dst_ip.c_str()); + int cube_id = rand() % CUBE_NUM; + const Fieldstat_tag_list_wrapper *shared_tag = shared_tags[cube_id]; + + int ret_add = fieldstat_hll_add_field(instance, cube_id, 0, cell_dimension.get_tag(), cell_dimension.get_tag_count(), item.get_tag(), item.get_tag_count()); + if (ret_add == FS_ERR_TOO_MANY_CELLS) { + continue; + } + EXPECT_EQ(ret_add, FS_OK); + count_map[shared_tag->to_string()][cell_dimension.to_string()] += 1; + } + + clock_t end = clock(); + printf("time: %lf\n", (double)(end - start) / CLOCKS_PER_SEC); + + for (int i = 0; i < CUBE_NUM; i++) { + delete shared_tags[i]; + } + + int *cube_ids; + int cube_num; + struct fieldstat *instance_in_focus = dest; + fieldstat_get_cubes(instance_in_focus, &cube_ids, &cube_num); + double sum_accuracy = 0; + for (int i = 0; i < cube_num; i++) { + struct field_list *shared_tag_out = fieldstat_cube_get_dimensions(instance_in_focus, cube_ids[i]); + + size_t cell_num; + struct field_list *cells; + fieldstat_cube_get_cells(instance_in_focus, cube_ids[i], &cells, &cell_num); + + std::vector<struct Fieldstat_tag_list_wrapper *> test_result; + for (size_t j = 0; j < cell_num; j++) { + test_result.push_back(new Fieldstat_tag_list_wrapper(&cells[j])); + } + + Fieldstat_tag_list_wrapper cube_dimension = Fieldstat_tag_list_wrapper(shared_tag_out); + double accuracy = test_cal_topk_accuracy(test_result, count_map[cube_dimension.to_string()]); + // printf("spreadsketch accuracy: %lf\n", accuracy); + sum_accuracy += accuracy; + + // CM sketch error + double est_total = 0; + double true_total = 0; + for (size_t j = 0; j < cell_num; j++) { + Fieldstat_tag_list_wrapper cell_dimension = Fieldstat_tag_list_wrapper(&cells[j]); + double value_true = count_map[cube_dimension.to_string()][cell_dimension.to_string()]; + double value_est = 0; + fieldstat_hll_get(instance_in_focus, cube_ids[i], &cells[j], 0, &value_est); + // printf("cube:%s, cell:%s, true:%lf, est:%lf\n", cube_dimension.to_string().c_str(), cell_dimension.to_string().c_str(), value_true, value_est); + + est_total += value_est; + true_total += value_true; + } + EXPECT_LE(abs(est_total - true_total) / true_total, 0.2); + // printf("spreadsketch Mean ratio e: %f\n", abs(est_total - true_total) / true_total); + + for (size_t j = 0; j < cell_num; j++) { + delete test_result[j]; + } + + fieldstat_field_list_arr_free(cells, cell_num); + fieldstat_field_list_arr_free(shared_tag_out, 1); + } + double mean_accuracy = sum_accuracy / cube_num; + EXPECT_GE(mean_accuracy, 0.7); + + free(cube_ids); + + fieldstat_free(master); + fieldstat_free(dest); + for (int i = 0; i < INSTANCE_NUM; i++) { + fieldstat_free(replica[i]); + } +} + +// issue： https://jira.geedge.net/browse/TSG-21192 +// 在reset后，所有项都是dying 状态，此时添加count = 0 的项，不能正常把dying pop掉，误以为sorted set 已满，出现添加失败（FS_ERR_TOO_MANY_CELLS）但是查不到任何cell 的情况。 TEST(Fuzz_test, add_and_reset_with_randomly_generated_flows_and_randomly_chosen_metric) { const int FLOW_NUM = 50000; @@ -335,40 +507,48 @@ TEST(Fuzz_test, add_and_reset_with_randomly_generated_flows_and_randomly_chosen_ fill_with_elephant_flows(tag_list_wrapper, FLOW_NUM); struct fieldstat *instance = fieldstat_new(); - int primary_metric_id = fieldstat_register_counter(instance, "counter"); - int counter2_id = fieldstat_register_counter(instance, "counter2"); - int cube_id = fieldstat_create_cube(instance,NULL,0,SAMPLING_MODE_TOPK, 1); // K = 1, just to increase the possibility of FS_ERR_TOO_MANY_CELLS + + int cube_id = fieldstat_cube_create(instance,NULL,0); + int primary_metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + int counter2_id = fieldstat_register_counter(instance, cube_id, "counter2"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 1, 0); // K = 1, just to increase the possibility of FS_ERR_TOO_MANY_CELLS + fieldstat_counter_incrby(instance, cube_id, primary_metric_id, tag_list_wrapper[0]->get_tag(), tag_list_wrapper[0]->get_tag_count(), 1); fieldstat_counter_incrby(instance, cube_id, counter2_id, tag_list_wrapper[0]->get_tag(), tag_list_wrapper[0]->get_tag_count(), 1); - struct fieldstat_tag_list tag_list_tmp = {NULL, 0}; + struct field_list tag_list_tmp = {NULL, 0}; for(int i = 0; i < FLOW_NUM; i++) { int using_id = rand() % 2 == 0 ? primary_metric_id : counter2_id; int ret = fieldstat_counter_incrby(instance, cube_id, using_id, tag_list_wrapper[i]->get_tag(), tag_list_wrapper[i]->get_tag_count(), 1); if (ret == FS_ERR_TOO_MANY_CELLS) { - struct fieldstat_tag_list *tag_list = NULL; + struct field_list *tag_list = NULL; size_t n_cell = 0; - fieldstat_get_cells_used_by_cube(instance, cube_id, &tag_list, &n_cell); + fieldstat_cube_get_cells(instance, cube_id, &tag_list, &n_cell); ASSERT_EQ(n_cell, 1); long long value; - tag_list_tmp.tag = (struct fieldstat_tag *)tag_list_wrapper[i]->get_tag(); - tag_list_tmp.n_tag = tag_list_wrapper[i]->get_tag_count(); - int counter_exist = fieldstat_counter_get(instance, cube_id, using_id, &tag_list_tmp, &value); - ASSERT_EQ(counter_exist, FS_ERR_INVALID_TAG); // the tag is not added to the cube - fieldstat_tag_list_arr_free(tag_list, n_cell); + tag_list_tmp.field = (struct field *)tag_list_wrapper[i]->get_tag(); + tag_list_tmp.n_field = tag_list_wrapper[i]->get_tag_count(); + int counter_exist = fieldstat_counter_get(instance, cube_id, &tag_list_tmp, using_id, &value); + ASSERT_EQ(counter_exist, FS_ERR_INVALID_DIMENSION); // the field is not added to the cube + fieldstat_field_list_arr_free(tag_list, n_cell); } if (i % 1000 == 0) { fieldstat_reset(instance); } } + + for (int i = 0; i < FLOW_NUM; i++) { + delete tag_list_wrapper[i]; + } + fieldstat_free(instance); } -TEST(Fuzz_test, simple_one_for_perf) +TEST(perf, simple_one_for_perf_topk) { const int CUBE_NUM = 5; const int FLOW_NUM = 50000; @@ -381,12 +561,14 @@ TEST(Fuzz_test, simple_one_for_perf) // init cube for (int i = 0; i < CUBE_NUM; i++) { shared_tags[i] = new Fieldstat_tag_list_wrapper("shared_tag", i); - int cube_id = fieldstat_create_cube(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count(), SAMPLING_MODE_TOPK, CELL_MAX); + int cube_id = fieldstat_cube_create(master, shared_tags[i]->get_tag(), shared_tags[i]->get_tag_count()); EXPECT_EQ(cube_id, i); + fieldstat_register_counter(master, cube_id, "topk"); + fieldstat_cube_set_sampling(master, cube_id, SAMPLING_MODE_TOPK, CELL_MAX, 0); } // init metric - fieldstat_register_counter(master, "topk"); - // all the possible tags + + // all the possible fields Fieldstat_tag_list_wrapper *tag_list_wrapper[FLOW_NUM]; fill_with_elephant_flows(tag_list_wrapper, FLOW_NUM); //all the possible operations @@ -398,15 +580,15 @@ TEST(Fuzz_test, simple_one_for_perf) struct fieldstat *instance = master; clock_t start = clock(); - printf("press any key to start\n"); + printf("press any key to start v46\n"); getchar(); for (int i = 0; i < TEST_ROUND; i++) { - const Fieldstat_tag_list_wrapper * tag = tag_list_wrapper[rand() % FLOW_NUM]; + const Fieldstat_tag_list_wrapper * field = tag_list_wrapper[rand() % FLOW_NUM]; int cube_id = rand() % CUBE_NUM; - (void)fieldstat_counter_incrby(instance, cube_id, 0, tag->get_tag(), tag->get_tag_count(), rand_nums[i]); + (void)fieldstat_counter_incrby(instance, cube_id, 0, field->get_tag(), field->get_tag_count(), rand_nums[i]); } clock_t end = clock(); @@ -422,10 +604,44 @@ TEST(Fuzz_test, simple_one_for_perf) fieldstat_free(master); } +TEST(perf, simple_one_for_perf_spreadsketch) +{ + const int CELL_MAX = 100; + const int TEST_ROUND = 500000; + struct fieldstat *instance = fieldstat_new(); + + int cube_id = fieldstat_cube_create(instance, &TEST_FIELD_STRING, 1); + fieldstat_register_hll(instance, cube_id, "hll", 6); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, CELL_MAX, 0); + + SpreadSketchZipfGenerator generator(1.0, CELL_MAX * 10); + Fieldstat_tag_list_wrapper *cell_dimension[TEST_ROUND]; + Fieldstat_tag_list_wrapper *items[TEST_ROUND]; + for (int i = 0; i < TEST_ROUND; i++) { + Flow flow = generator.next(); + cell_dimension[i] = new Fieldstat_tag_list_wrapper("src_ip", flow.src_ip.c_str()); + items[i] = new Fieldstat_tag_list_wrapper("dst_ip", flow.dst_ip.c_str()); + } + + clock_t start = clock(); + printf("press any key to start \n"); + getchar(); + + for (int i = 0; i < TEST_ROUND; i++) { + fieldstat_hll_add_field(instance, cube_id, 0, cell_dimension[i]->get_tag(), cell_dimension[i]->get_tag_count(), items[i]->get_tag(), items[i]->get_tag_count()); + } + + clock_t end = clock(); + printf("time: %lf second\n", (double)(end - start) / CLOCKS_PER_SEC); + + fieldstat_free(instance); +} int main(int argc, char *argv[]) { testing::InitGoogleTest(&argc, argv); - testing::GTEST_FLAG(filter) = "-Fuzz_test.simple_one_for_perf"; + // testing::GTEST_FLAG(filter) = "*many_instance_random_flow_unregister_calibrate_reset_fork_merge_spreadsketch"; + testing::GTEST_FLAG(filter) = "-perf.*"; + return RUN_ALL_TESTS(); }
\ No newline at end of file diff --git a/test/test_merge.cpp b/test/test_merge.cpp index a37dff9..46583d1 100644 --- a/test/test_merge.cpp +++ b/test/test_merge.cpp @@ -2,6 +2,7 @@ #include <gtest/gtest.h> #include <set> #include <unordered_map> +#include <unordered_set> #include "fieldstat.h" #include "utils.hpp" @@ -31,64 +32,37 @@ double test_cal_accuracy_given_expected_key(vector<struct Fieldstat_tag_list_wra return test_cal_topk_accuracy(test_result, countMap); } -long long merge_test_fieldstat_counter_get(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tag_list = &TEST_TAG_LIST_STRING) +long long merge_test_fieldstat_counter_get(const struct fieldstat *instance, int cube_id, int metric_id, const struct field_list *tag_list = &TEST_FIELD_LIST_STRING) { long long ret = 0; - fieldstat_counter_get(instance, cube_id, metric_id, tag_list, &ret); + fieldstat_counter_get(instance, cube_id, tag_list, metric_id, &ret); return ret; } -TEST(unit_test_merge, test_metric_name_mapping_with_new_metric_on_existing_cube) +double merge_test_fieldstat_hll_get(const struct fieldstat *instance, int cube_id, int metric_id, const struct field_list *tag_list = &TEST_FIELD_LIST_STRING) { - struct fieldstat *instance = fieldstat_new(); - - int cube_id1 = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int metric_id_1_0 = fieldstat_register_counter(instance, "metric_name cube1 cube2"); - int metric_id_1_1 = fieldstat_register_counter(instance, "shared name"); - fieldstat_counter_incrby(instance, cube_id1, metric_id_1_0, &TEST_TAG_STRING, 1, 1); - fieldstat_counter_incrby(instance, cube_id1, metric_id_1_1, &TEST_TAG_STRING, 1, 2); - int cube_id2 = fieldstat_create_cube(instance, &TEST_TAG_INT, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - fieldstat_counter_incrby(instance, cube_id2, metric_id_1_0, &TEST_TAG_STRING, 1, 3); - - struct fieldstat *instance_dest = fieldstat_new(); - int cube_id_dest = fieldstat_create_cube(instance_dest, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - (void)fieldstat_register_counter(instance_dest, "shared name"); - // shared name metric is not operated on cube_id_dest - - EXPECT_EQ(fieldstat_merge(instance_dest, instance), FS_OK); - - int *cube_id; - int n_cube; - fieldstat_get_cubes(instance_dest, &cube_id, &n_cube); - EXPECT_TRUE(n_cube == 2); - EXPECT_TRUE(cube_id[0] == cube_id_dest); - - int *metric_ids; - size_t n_metrics; - fieldstat_get_metrics_used_by_cube(instance_dest, cube_id_dest, &metric_ids, &n_metrics); - EXPECT_EQ(n_metrics, 2); - EXPECT_STREQ(fieldstat_get_metric_name(instance_dest, metric_ids[0]), "shared name"); - EXPECT_STREQ(fieldstat_get_metric_name(instance_dest, metric_ids[1]), "metric_name cube1 cube2"); - - EXPECT_EQ(merge_test_fieldstat_counter_get(instance_dest, cube_id_dest, 0), 2); // shared name - EXPECT_EQ(merge_test_fieldstat_counter_get(instance_dest, cube_id_dest, 1), 1); // metric_name cube1 cube2 on cube1 - EXPECT_EQ(merge_test_fieldstat_counter_get(instance_dest, cube_id[1], 1), 3); // metric_name cube1 cube2 on cube2 + double ret = 0; + fieldstat_hll_get(instance, cube_id, tag_list, metric_id, &ret); + return ret; +} - fieldstat_free(instance); - fieldstat_free(instance_dest); - free(cube_id); - free(metric_ids); +int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k, int primary_metric_id=0) +{ + assert(mode == SAMPLING_MODE_COMPREHENSIVE); + int ret = fieldstat_cube_create(instance, dimensions, n_dimensions); + fieldstat_cube_set_sampling(instance, ret, mode, k, primary_metric_id); + return ret; } TEST(unit_test_merge, cube_shared_tag_mapping_with_new_cube) { struct fieldstat *instance = fieldstat_new(); - (void)fieldstat_create_cube(instance, &TEST_TAG_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int cube_id2 = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int metric_id = fieldstat_register_counter(instance, "metric in cube 2"); - fieldstat_counter_incrby(instance, cube_id2, metric_id, &TEST_TAG_STRING, 1, 1); + (void)test_fieldstat_cube_create(instance, &TEST_FIELD_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id2 = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int metric_id = fieldstat_register_counter(instance,cube_id2,"metric in cube 2"); + fieldstat_counter_incrby(instance, cube_id2, metric_id, &TEST_FIELD_STRING, 1, 1); struct fieldstat *instance_dest = fieldstat_new(); - int cube_id_dest = fieldstat_create_cube(instance_dest, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id_dest = test_fieldstat_cube_create(instance_dest, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); fieldstat_merge(instance_dest, instance); @@ -121,8 +95,8 @@ TEST(unit_test_merge, empty_instance) TEST(unit_test_merge, new_cube_and_metric_to_empty_comprehensive) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - fieldstat_register_counter(instance, "metric_name"); + test_fieldstat_cube_create(instance, &TEST_FIELD_INT, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + fieldstat_register_counter(instance, 0, "metric_name"); struct fieldstat *instance_dest = fieldstat_new(); @@ -132,7 +106,7 @@ TEST(unit_test_merge, new_cube_and_metric_to_empty_comprehensive) int n_cube; fieldstat_get_cubes(instance_dest, &cube_id_dest, &n_cube); EXPECT_TRUE(n_cube == 1); - EXPECT_STREQ(fieldstat_get_metric_name(instance_dest, 0), "metric_name"); + EXPECT_STREQ(fieldstat_get_metric_name(instance_dest, cube_id_dest[0], 0), "metric_name"); fieldstat_free(instance); fieldstat_free(instance_dest); @@ -142,47 +116,47 @@ TEST(unit_test_merge, new_cube_and_metric_to_empty_comprehensive) TEST(unit_test_merge, new_cell_on_existing_cube_and_metric_comprehensive) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int metric_id = fieldstat_register_counter(instance, "metric_name"); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int metric_id = fieldstat_register_counter(instance, 0, "metric_name"); struct fieldstat *instance_dest = fieldstat_new(); fieldstat_merge(instance_dest, instance); - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, 10086); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_STRING, 1, 10086); fieldstat_merge(instance_dest, instance); int *cube_id_dest; int n_cube; fieldstat_get_cubes(instance_dest, &cube_id_dest, &n_cube); EXPECT_TRUE(n_cube == 1); + EXPECT_STREQ(fieldstat_get_metric_name(instance_dest, cube_id_dest[0], 0), "metric_name"); free(cube_id_dest); - EXPECT_STREQ(fieldstat_get_metric_name(instance_dest, 0), "metric_name"); long long measure = merge_test_fieldstat_counter_get(instance, cube_id, metric_id); EXPECT_EQ(measure, 10086); - struct fieldstat_tag_list *tag_list = NULL; + struct field_list *tag_list = NULL; size_t n_cell = 0; - fieldstat_get_cells_used_by_metric(instance, cube_id, metric_id, &tag_list, &n_cell); + fieldstat_cube_get_cells(instance, cube_id, &tag_list, &n_cell); EXPECT_EQ(n_cell, 1); - EXPECT_EQ(tag_list->n_tag, 1); - EXPECT_STREQ(tag_list->tag[0].key, TEST_TAG_STRING.key); + EXPECT_EQ(tag_list->n_field, 1); + EXPECT_STREQ(tag_list->field[0].key, TEST_FIELD_STRING.key); fieldstat_free(instance); fieldstat_free(instance_dest); - fieldstat_tag_list_arr_free(tag_list, n_cell); + fieldstat_field_list_arr_free(tag_list, n_cell); } TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_comprehensive) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int metric_id = fieldstat_register_counter(instance, "metric_name"); - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, 5); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int metric_id = fieldstat_register_counter(instance, cube_id, "metric_name"); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_STRING, 1, 5); struct fieldstat *instance_dest = fieldstat_new(); fieldstat_merge(instance_dest, instance); fieldstat_merge(instance_dest, instance); - EXPECT_STREQ(fieldstat_get_metric_name(instance_dest, 0), "metric_name"); + EXPECT_STREQ(fieldstat_get_metric_name(instance_dest, cube_id, 0), "metric_name"); long long measure = merge_test_fieldstat_counter_get(instance_dest, cube_id, metric_id); EXPECT_EQ(measure, 10); @@ -193,61 +167,61 @@ TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_comprehens TEST(unit_test_merge, new_too_many_cells_on_one_metric_given_source_cube_reset_and_get_different_cube_comprehensive) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2); // limit is 2 - int metric_id = fieldstat_register_counter(instance, "metric name"); - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, 1); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2); // limit is 2 + int metric_id = fieldstat_register_counter(instance, cube_id, "metric name"); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_STRING, 1, 1); struct fieldstat *instance_dest = fieldstat_new(); fieldstat_merge(instance_dest, instance); fieldstat_reset(instance); - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 2); // 2nd cell - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_DOUBLE, 1, 3); // 3rd cell, exceeding the limit 2 + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, 2); // 2nd cell + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_DOUBLE, 1, 3); // 3rd cell, exceeding the limit 2 fieldstat_merge(instance_dest, instance); - struct fieldstat_tag_list *tag_list = NULL; + struct field_list *tag_list = NULL; size_t n_cell = 0; - fieldstat_get_cells_used_by_metric(instance_dest, 0, 0, &tag_list, &n_cell); + fieldstat_cube_get_cells(instance_dest, 0, &tag_list, &n_cell); EXPECT_EQ(n_cell, 2); EXPECT_EQ(merge_test_fieldstat_counter_get(instance_dest, 0, 0, &tag_list[0]), 1); EXPECT_EQ(merge_test_fieldstat_counter_get(instance_dest, 0, 0, &tag_list[1]), 2); fieldstat_free(instance); fieldstat_free(instance_dest); - fieldstat_tag_list_arr_free(tag_list, n_cell); + fieldstat_field_list_arr_free(tag_list, n_cell); } TEST(unit_test_merge, new_too_many_cells_on_multiple_metric_given_source_cube_reset_and_get_different_cube_comprehensive) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2); - int metric_id1 = fieldstat_register_counter(instance, "metric name1"); - int metric_id2 = fieldstat_register_counter(instance, "metric name2"); - fieldstat_counter_incrby(instance, cube_id, metric_id1, &TEST_TAG_STRING, 1, 1); // 1st cell on metric name1 + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2); + int metric_id1 = fieldstat_register_counter(instance, cube_id, "metric name1"); + int metric_id2 = fieldstat_register_counter(instance, cube_id, "metric name2"); + fieldstat_counter_incrby(instance, cube_id, metric_id1, &TEST_FIELD_STRING, 1, 1); // 1st cell on metric name1 struct fieldstat *instance_dest = fieldstat_new(); fieldstat_merge(instance_dest, instance); fieldstat_reset(instance); - int metric_id3 = fieldstat_register_counter(instance, "metric name3"); - fieldstat_counter_incrby(instance, cube_id, metric_id3, &TEST_TAG_INT, 1, 2); // 2nd cell on metric name3, this is a metric dest dont have - fieldstat_counter_incrby(instance, cube_id, metric_id2, &TEST_TAG_DOUBLE, 1, 3); // 3nd cell on metric name2 + int metric_id3 = fieldstat_register_counter(instance, cube_id, "metric name3"); + fieldstat_counter_incrby(instance, cube_id, metric_id3, &TEST_FIELD_INT, 1, 2); // 2nd cell on metric name3, this is a metric dest dont have + fieldstat_counter_incrby(instance, cube_id, metric_id2, &TEST_FIELD_DOUBLE, 1, 3); // 3nd cell on metric name2 fieldstat_merge(instance_dest, instance); - struct fieldstat_tag_list *tag_list = NULL; - size_t n_cell = 0; - fieldstat_get_cells_used_by_metric(instance_dest, 0, metric_id1, &tag_list, &n_cell); - EXPECT_EQ(n_cell, 1); - EXPECT_STREQ(tag_list->tag[0].key, TEST_TAG_STRING.key); - fieldstat_tag_list_arr_free(tag_list, n_cell); - - fieldstat_get_cells_used_by_metric(instance_dest, 0, metric_id2, &tag_list, &n_cell); // 3nd cell failed to merge - EXPECT_EQ(n_cell, 0); + int *metric_ids = NULL; + size_t n_metrics = 0; + fieldstat_get_metric_in_cell(instance_dest, 0, &TEST_FIELD_LIST_STRING, &metric_ids, &n_metrics); + EXPECT_EQ(n_metrics, 1); + EXPECT_STREQ(fieldstat_get_metric_name(instance_dest, 0, metric_ids[0]), "metric name1"); + free(metric_ids); - fieldstat_get_cells_used_by_metric(instance_dest, 0, metric_id3, &tag_list, &n_cell); - EXPECT_EQ(n_cell, 1); - EXPECT_STREQ(tag_list->tag[0].key, TEST_TAG_INT.key); - fieldstat_tag_list_arr_free(tag_list, n_cell); + fieldstat_get_metric_in_cell(instance_dest, 0, &TEST_FIELD_LIST_INT, &metric_ids, &n_metrics); + EXPECT_EQ(n_metrics, 1); + EXPECT_STREQ(fieldstat_get_metric_name(instance_dest, 0, metric_ids[0]), "metric name3"); + free(metric_ids); + // 3nd cell failed to merge, because max sampling is 2 + fieldstat_get_metric_in_cell(instance_dest, 0, &TEST_FIELD_LIST_DOUBLE, &metric_ids, &n_metrics); + EXPECT_EQ(n_metrics, 0); fieldstat_free(instance); fieldstat_free(instance_dest); @@ -256,8 +230,9 @@ TEST(unit_test_merge, new_too_many_cells_on_multiple_metric_given_source_cube_re TEST(unit_test_merge, new_cube_and_metric_to_empty_topk) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT, 1, SAMPLING_MODE_TOPK, 10); - fieldstat_register_counter(instance, "metric_name"); + fieldstat_cube_create(instance, &TEST_FIELD_INT, 1); + fieldstat_register_counter(instance, 0, "metric_name"); + fieldstat_cube_set_sampling(instance, 0, SAMPLING_MODE_TOPK, 10, 0); struct fieldstat *instance_dest = fieldstat_new(); @@ -267,7 +242,7 @@ TEST(unit_test_merge, new_cube_and_metric_to_empty_topk) int n_cube; fieldstat_get_cubes(instance_dest, &cube_id_dest, &n_cube); EXPECT_TRUE(n_cube == 1); - EXPECT_STREQ(fieldstat_get_metric_name(instance_dest, 0), "metric_name"); + EXPECT_STREQ(fieldstat_get_metric_name(instance_dest, cube_id_dest[0], 0), "metric_name"); fieldstat_free(instance); fieldstat_free(instance_dest); @@ -277,41 +252,43 @@ TEST(unit_test_merge, new_cube_and_metric_to_empty_topk) TEST(unit_test_merge, new_cell_on_existing_cube_and_metric_topk) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); - int metric_id = fieldstat_register_counter(instance, "metric_name"); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_counter(instance, cube_id, "metric_name"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, 0); struct fieldstat *instance_dest = fieldstat_new(); fieldstat_merge(instance_dest, instance); - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, 10086); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_STRING, 1, 10086); fieldstat_merge(instance_dest, instance); int *cube_id_dest; int n_cube; fieldstat_get_cubes(instance_dest, &cube_id_dest, &n_cube); EXPECT_TRUE(n_cube == 1); + EXPECT_STREQ(fieldstat_get_metric_name(instance_dest,cube_id_dest[0], 0), "metric_name"); free(cube_id_dest); - EXPECT_STREQ(fieldstat_get_metric_name(instance_dest, 0), "metric_name"); long long measure = merge_test_fieldstat_counter_get(instance, cube_id, metric_id); EXPECT_EQ(measure, 10086); - struct fieldstat_tag_list *tag_list = NULL; + struct field_list *tag_list = NULL; size_t n_cell = 0; - fieldstat_get_cells_used_by_metric(instance, cube_id, metric_id, &tag_list, &n_cell); + fieldstat_cube_get_cells(instance, cube_id, &tag_list, &n_cell); EXPECT_EQ(n_cell, 1); - EXPECT_EQ(tag_list->n_tag, 1); - EXPECT_STREQ(tag_list->tag[0].key, TEST_TAG_STRING.key); + EXPECT_EQ(tag_list->n_field, 1); + EXPECT_STREQ(tag_list->field[0].key, TEST_FIELD_STRING.key); fieldstat_free(instance); fieldstat_free(instance_dest); - fieldstat_tag_list_arr_free(tag_list, n_cell); + fieldstat_field_list_arr_free(tag_list, n_cell); } TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_topk) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); - int metric_id = fieldstat_register_counter(instance, "metric_name"); - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, 5); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_counter(instance, cube_id, "metric_name"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, 0); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_STRING, 1, 5); struct fieldstat *instance_dest = fieldstat_new(); fieldstat_merge(instance_dest, instance); @@ -322,17 +299,17 @@ TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_topk) fieldstat_get_cubes(instance_dest, &cube_id_dest, &n_cube); EXPECT_TRUE(n_cube == 1); int ret_cube_id = cube_id_dest[0]; + EXPECT_STREQ(fieldstat_get_metric_name(instance_dest,cube_id_dest[0], 0), "metric_name"); free(cube_id_dest); - EXPECT_STREQ(fieldstat_get_metric_name(instance_dest, 0), "metric_name"); - struct fieldstat_tag_list *tag_list = NULL; + struct field_list *tag_list = NULL; size_t n_cell = 0; - fieldstat_get_cells_used_by_metric(instance_dest, ret_cube_id, 0, &tag_list, &n_cell); + fieldstat_cube_get_cells(instance_dest, ret_cube_id, &tag_list, &n_cell); EXPECT_EQ(n_cell, 1); long long measure = merge_test_fieldstat_counter_get(instance_dest, cube_id, metric_id, &tag_list[0]); EXPECT_EQ(measure, 10); - fieldstat_tag_list_arr_free(tag_list, n_cell); + fieldstat_field_list_arr_free(tag_list, n_cell); fieldstat_free(instance); fieldstat_free(instance_dest); } @@ -340,36 +317,38 @@ TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_topk) TEST(unit_test_merge, new_too_many_cells_on_one_metric_given_source_cube_reset_and_get_different_cube_topk) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 2); - int metric_id = fieldstat_register_counter(instance, "metric name"); - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, 1); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_counter(instance, cube_id, "metric name"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 2, 0); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_STRING, 1, 1); struct fieldstat *instance_dest = fieldstat_new(); fieldstat_merge(instance_dest, instance); fieldstat_reset(instance); - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 2); // 2nd cell - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_DOUBLE, 1, 3); // 3rd cell,bigger than the others, so keep it + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, 2); // 2nd cell + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_DOUBLE, 1, 3); // 3rd cell,bigger than the others, so keep it fieldstat_merge(instance_dest, instance); - struct fieldstat_tag_list *tag_list = NULL; + struct field_list *tag_list = NULL; size_t n_cell = 0; - fieldstat_get_cells_used_by_metric(instance_dest, 0, 0, &tag_list, &n_cell); + fieldstat_cube_get_cells(instance_dest, 0, &tag_list, &n_cell); EXPECT_EQ(n_cell, 2); EXPECT_EQ(merge_test_fieldstat_counter_get(instance_dest, 0, 0, &tag_list[0]), 3); EXPECT_EQ(merge_test_fieldstat_counter_get(instance_dest, 0, 0, &tag_list[1]), 2); fieldstat_free(instance); fieldstat_free(instance_dest); - fieldstat_tag_list_arr_free(tag_list, n_cell); + fieldstat_field_list_arr_free(tag_list, n_cell); } -struct fieldstat *test_push_flows(vector<Fieldstat_tag_list_wrapper *> &flows_in_test, int K, long long count = 1) +struct fieldstat *topk_test_push_flows(vector<Fieldstat_tag_list_wrapper *> &flows_in_test, int K, long long count = 1) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, K); - int metric_id = fieldstat_register_counter(instance, "metric name"); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_counter(instance, cube_id, "metric name"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, K, 0); for (size_t i = 0; i < flows_in_test.size(); i++) { fieldstat_counter_incrby(instance, cube_id, metric_id, flows_in_test[i]->get_tag(), flows_in_test[i]->get_tag_count(), count); } @@ -380,14 +359,14 @@ TEST(unit_test_merge, merge_accuracy_test_with_K_large_enough_topk) { int K = 100; vector<Fieldstat_tag_list_wrapper *> flows_in_src = test_gen_topk_flows(K, K); - struct fieldstat *instance_src = test_push_flows(flows_in_src, K); + struct fieldstat *instance_src = topk_test_push_flows(flows_in_src, K); vector<Fieldstat_tag_list_wrapper *> flows_in_dest = test_gen_topk_flows(K, K); - struct fieldstat *instance_dest = test_push_flows(flows_in_dest, K); + struct fieldstat *instance_dest = topk_test_push_flows(flows_in_dest, K); fieldstat_merge(instance_dest, instance_src); - struct fieldstat_tag_list *tag_list = NULL; + struct field_list *tag_list = NULL; size_t n_cell = 0; - fieldstat_get_cells_used_by_metric(instance_dest, 0, 0, &tag_list, &n_cell); + fieldstat_cube_get_cells(instance_dest, 0, &tag_list, &n_cell); vector<Fieldstat_tag_list_wrapper *> flows_in_merged; for (size_t i = 0; i < n_cell; i++) { @@ -400,7 +379,7 @@ TEST(unit_test_merge, merge_accuracy_test_with_K_large_enough_topk) fieldstat_free(instance_src); fieldstat_free(instance_dest); - fieldstat_tag_list_arr_free(tag_list, n_cell); + fieldstat_field_list_arr_free(tag_list, n_cell); for (size_t i = 0; i < flows_in_merged.size(); i++) { delete flows_in_merged[i]; } @@ -409,23 +388,23 @@ TEST(unit_test_merge, merge_accuracy_test_with_K_large_enough_topk) } } -TEST(unit_test_merge, merge_accuracy_test_gen_dest_full_all_inserted_given_src_flows_larger) +TEST(unit_test_merge, merge_accuracy_test_gen_dest_full_all_inserted_given_src_flows_larger_topk) { int K = 1000; vector<Fieldstat_tag_list_wrapper *> flows_in_src = test_gen_topk_flows(10000, K); - struct fieldstat *instance_src = test_push_flows(flows_in_src, K, 1000); // 1000 times larger than dest 1 + struct fieldstat *instance_src = topk_test_push_flows(flows_in_src, K, 1000); // 1000 times larger than dest 1 vector<Fieldstat_tag_list_wrapper *> flows_in_dest; for (int i = 0; i < K; i++) { Fieldstat_tag_list_wrapper *tmp = new Fieldstat_tag_list_wrapper("flows in dest", to_string(i).c_str()); flows_in_dest.push_back(tmp); } - struct fieldstat *instance_dest = test_push_flows(flows_in_dest, K, 1); + struct fieldstat *instance_dest = topk_test_push_flows(flows_in_dest, K, 1); fieldstat_merge(instance_dest, instance_src); - struct fieldstat_tag_list *tag_list = NULL; + struct field_list *tag_list = NULL; size_t n_cell = 0; - fieldstat_get_cells_used_by_metric(instance_dest, 0, 0, &tag_list, &n_cell); + fieldstat_cube_get_cells(instance_dest, 0, &tag_list, &n_cell); vector<Fieldstat_tag_list_wrapper *> flows_in_merged; for (size_t i = 0; i < n_cell; i++) { flows_in_merged.push_back(new Fieldstat_tag_list_wrapper(&tag_list[i])); @@ -438,7 +417,7 @@ TEST(unit_test_merge, merge_accuracy_test_gen_dest_full_all_inserted_given_src_f fieldstat_free(instance_src); fieldstat_free(instance_dest); - fieldstat_tag_list_arr_free(tag_list, n_cell); + fieldstat_field_list_arr_free(tag_list, n_cell); for (size_t i = 0; i < flows_in_merged.size(); i++) { delete flows_in_merged[i]; } @@ -447,18 +426,18 @@ TEST(unit_test_merge, merge_accuracy_test_gen_dest_full_all_inserted_given_src_f } } -TEST(unit_test_merge, merge_accuracy_test_gen_dest_full_some_inserted_and_some_merged_and_some_fail_to_add) +TEST(unit_test_merge, merge_accuracy_test_gen_dest_full_some_inserted_and_some_merged_and_some_fail_to_add_topk) { int K = 100; - vector<Fieldstat_tag_list_wrapper *> flows_in_src = test_gen_topk_flows(10000, K + 50); // let elephant flows in src and dest different - struct fieldstat *instance_src = test_push_flows(flows_in_src, K); - vector<Fieldstat_tag_list_wrapper *> flows_in_dest = test_gen_topk_flows(10000, K + 50); - struct fieldstat *instance_dest = test_push_flows(flows_in_dest, K); + vector<Fieldstat_tag_list_wrapper *> flows_in_src = test_gen_topk_flows(30000, K + 50); // let elephant flows in src and dest different + struct fieldstat *instance_src = topk_test_push_flows(flows_in_src, K); + vector<Fieldstat_tag_list_wrapper *> flows_in_dest = test_gen_topk_flows(30000, K + 50); + struct fieldstat *instance_dest = topk_test_push_flows(flows_in_dest, K); fieldstat_merge(instance_dest, instance_src); - struct fieldstat_tag_list *tag_list = NULL; + struct field_list *tag_list = NULL; size_t n_cell = 0; - fieldstat_get_cells_used_by_metric(instance_dest, 0, 0, &tag_list, &n_cell); + fieldstat_cube_get_cells(instance_dest, 0, &tag_list, &n_cell); vector<Fieldstat_tag_list_wrapper *> flows_in_merged; for (size_t i = 0; i < n_cell; i++) { flows_in_merged.push_back(new Fieldstat_tag_list_wrapper(&tag_list[i])); @@ -466,12 +445,12 @@ TEST(unit_test_merge, merge_accuracy_test_gen_dest_full_some_inserted_and_some_m flows_in_dest.insert(flows_in_dest.end(), std::make_move_iterator(flows_in_src.begin()), std::make_move_iterator(flows_in_src.end())); double accuracy = test_cal_accuracy_given_expected_key(flows_in_dest, flows_in_merged); - EXPECT_GE(accuracy, 0.87); // by heavy keeper benchmark, with K = 100, merging result should be about 0.96, for adding the flows will also cause some inaccuracy, so here we set 0.93 + EXPECT_GE(accuracy, 0.87); printf("merge_accuracy_test_gen_dest_full_some_inserted_and_some_merged_and_some_fail_to_add accuracy is %lf\n", accuracy); fieldstat_free(instance_src); fieldstat_free(instance_dest); - fieldstat_tag_list_arr_free(tag_list, n_cell); + fieldstat_field_list_arr_free(tag_list, n_cell); for (size_t i = 0; i < flows_in_merged.size(); i++) { delete flows_in_merged[i]; } @@ -483,23 +462,24 @@ TEST(unit_test_merge, merge_accuracy_test_gen_dest_full_some_inserted_and_some_m TEST(unit_test_merge, primary_metric_has_no_value) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 2); - int metric_primary = fieldstat_register_counter(instance, "primary"); - int metric_operated = fieldstat_register_counter(instance, "operated"); - fieldstat_counter_incrby(instance, cube_id, metric_operated, &TEST_TAG_STRING, 1, 1); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_primary = fieldstat_register_counter(instance, cube_id, "primary"); + int metric_operated = fieldstat_register_counter(instance, cube_id, "operated"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 2, metric_primary); + fieldstat_counter_incrby(instance, cube_id, metric_operated, &TEST_FIELD_STRING, 1, 1); struct fieldstat *instance_dest = fieldstat_new(); fieldstat_merge(instance_dest, instance); fieldstat_merge(instance_dest, instance); - struct fieldstat_tag_list *tag_list = NULL; + struct field_list *tag_list = NULL; size_t n_cell = 0; - fieldstat_get_cells_used_by_metric(instance_dest, 0, metric_primary, &tag_list, &n_cell); - EXPECT_EQ(n_cell, 0); - - fieldstat_get_cells_used_by_metric(instance_dest, 0, metric_operated, &tag_list, &n_cell); + fieldstat_cube_get_cells(instance_dest, 0, &tag_list, &n_cell); EXPECT_EQ(n_cell, 1); + EXPECT_STREQ(tag_list[0].field[0].key, TEST_FIELD_STRING.key); + EXPECT_EQ(merge_test_fieldstat_counter_get(instance_dest, 0, metric_operated, &tag_list[0]), 2); - fieldstat_tag_list_arr_free(tag_list, n_cell); + EXPECT_EQ(merge_test_fieldstat_counter_get(instance_dest, 0, metric_primary, &tag_list[0]), 0); + fieldstat_field_list_arr_free(tag_list, n_cell); fieldstat_free(instance); fieldstat_free(instance_dest); @@ -508,42 +488,254 @@ TEST(unit_test_merge, primary_metric_has_no_value) TEST(unit_test_merge, primary_metric_id_different) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 2); - int metric_primary = fieldstat_register_counter(instance, "primary"); - int metric_2 = fieldstat_register_counter(instance, "2"); - fieldstat_counter_incrby(instance, cube_id, metric_primary, &TEST_TAG_STRING, 1, 100); - fieldstat_counter_incrby(instance, cube_id, metric_2, &TEST_TAG_STRING, 1, 1); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_primary = fieldstat_register_counter(instance, cube_id, "primary"); + int metric_2 = fieldstat_register_counter(instance, cube_id, "2"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 2, metric_primary); + + fieldstat_counter_incrby(instance, cube_id, metric_primary, &TEST_FIELD_STRING, 1, 100); + fieldstat_counter_incrby(instance, cube_id, metric_2, &TEST_FIELD_STRING, 1, 1); struct fieldstat *instance_dst = fieldstat_new(); - int cube_id_dst = fieldstat_create_cube(instance_dst, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 2); - fieldstat_register_counter(instance_dst, "2"); - int metric_primary_dst = fieldstat_register_counter(instance_dst, "primary"); - fieldstat_cube_set_primary_metric(instance_dst, cube_id_dst, metric_primary_dst); + int cube_id_dst = fieldstat_cube_create(instance_dst, &TEST_SHARED_TAG, 1); + fieldstat_register_counter(instance_dst, cube_id_dst, "2"); + int metric_primary_dst = fieldstat_register_counter(instance_dst, cube_id_dst, "primary"); + fieldstat_cube_set_sampling(instance_dst, cube_id_dst, SAMPLING_MODE_TOPK, 2, metric_primary_dst); + + EXPECT_EQ(fieldstat_merge(instance_dst, instance), FS_ERR_DIFFERENT_CONFIGURATION_FOR_SAME_CUBE); + + fieldstat_free(instance); + fieldstat_free(instance_dst); +} + +TEST(unit_test_merge, new_cube_and_metric_to_empty_spreadsketch) { + struct fieldstat *instance = fieldstat_new(); + fieldstat_cube_create(instance, &TEST_FIELD_INT, 1); + fieldstat_register_hll(instance, 0, "metric", 6); + fieldstat_cube_set_sampling(instance, 0, SAMPLING_MODE_TOP_CARDINALITY, 10, 0); + + struct fieldstat *instance_dest = fieldstat_new(); + fieldstat_merge(instance_dest, instance); + + int *cube_id_dest; + int n_cube; + fieldstat_get_cubes(instance_dest, &cube_id_dest, &n_cube); + EXPECT_TRUE(n_cube == 1); + EXPECT_STREQ(fieldstat_get_metric_name(instance_dest, cube_id_dest[0], 0), "metric"); + + free(cube_id_dest); + fieldstat_free(instance); + fieldstat_free(instance_dest); +} + +TEST(unit_test_merge, new_cell_on_existing_cube_and_metric_spreadsketch) { + struct fieldstat *instance = fieldstat_new(); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_hll(instance, cube_id, "metric", 6); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 10, 0); + struct fieldstat *instance_dest = fieldstat_new(); + fieldstat_merge(instance_dest, instance); + + fieldstat_hll_add(instance, cube_id, metric_id, &TEST_FIELD_STRING, 1, "1", 1); + fieldstat_hll_add(instance, cube_id, metric_id, &TEST_FIELD_STRING, 1, "2", 1); + fieldstat_merge(instance_dest, instance); - fieldstat_merge(instance_dst, instance); + int *cube_id_dest; + int n_cube; + fieldstat_get_cubes(instance_dest, &cube_id_dest, &n_cube); + EXPECT_TRUE(n_cube == 1); + EXPECT_STREQ(fieldstat_get_metric_name(instance_dest, cube_id_dest[0], 0), "metric"); + long long measure = merge_test_fieldstat_hll_get(instance, cube_id, metric_id); + EXPECT_NEAR(measure, 2, 0.3); - struct fieldstat_tag_list *tag_list = NULL; + struct field_list *tag_list = NULL; size_t n_cell = 0; - fieldstat_get_cells_used_by_metric(instance_dst, 0, metric_primary, &tag_list, &n_cell); + fieldstat_cube_get_cells(instance, cube_id, &tag_list, &n_cell); EXPECT_EQ(n_cell, 1); - int *metric_ids; - size_t n_metrics; - fieldstat_get_metrics(instance_dst, &metric_ids, &n_metrics); - EXPECT_EQ(n_metrics, 2); - EXPECT_STREQ(fieldstat_get_metric_name(instance_dst, metric_ids[0]), "2"); - EXPECT_STREQ(fieldstat_get_metric_name(instance_dst, metric_ids[1]), "primary"); - - EXPECT_EQ(merge_test_fieldstat_counter_get(instance_dst, 0, metric_ids[1], &tag_list[0]), 100); - EXPECT_EQ(merge_test_fieldstat_counter_get(instance_dst, 0, metric_ids[0], &tag_list[0]), 1); - fieldstat_tag_list_arr_free(tag_list, n_cell); - free(metric_ids); + EXPECT_EQ(tag_list->n_field, 1); + EXPECT_STREQ(tag_list->field[0].key, TEST_FIELD_STRING.key); + + free(cube_id_dest); fieldstat_free(instance); - fieldstat_free(instance_dst); + fieldstat_free(instance_dest); + fieldstat_field_list_arr_free(tag_list, n_cell); } +TEST(unit_test_merge, merge_existing_cell_on_existing_cube_and_metric_spreadsketch) { + struct fieldstat *instance = fieldstat_new(); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_hll(instance, cube_id, "metric", 6); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 10, 0); + fieldstat_hll_add(instance, cube_id, metric_id, &TEST_FIELD_STRING, 1, "1", 1); + struct fieldstat *instance_dest = fieldstat_new(); + + fieldstat_merge(instance_dest, instance); + fieldstat_merge(instance_dest, instance); + fieldstat_hll_add(instance, cube_id, metric_id, &TEST_FIELD_STRING, 1, "2", 1); + fieldstat_merge(instance_dest, instance); + + struct field_list *tag_list = NULL; + size_t n_cell = 0; + fieldstat_cube_get_cells(instance_dest, cube_id, &tag_list, &n_cell); + EXPECT_EQ(n_cell, 1); + double value = merge_test_fieldstat_hll_get(instance_dest, cube_id, metric_id, &tag_list[0]); + EXPECT_NEAR(value, 2, 0.3); + + fieldstat_free(instance); + fieldstat_free(instance_dest); + fieldstat_field_list_arr_free(tag_list, n_cell); +} + +TEST(unit_test_merge, new_too_many_cells_on_one_metric_given_source_cube_reset_and_get_different_cube_spreadsketch) { + struct fieldstat *instance = fieldstat_new(); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_hll(instance, cube_id, "metric", 6); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 2, 0); + fieldstat_hll_add(instance, cube_id, metric_id, &TEST_FIELD_STRING, 1, "1", 1); + struct fieldstat *instance_dest = fieldstat_new(); + fieldstat_merge(instance_dest, instance); + + fieldstat_reset(instance); + fieldstat_hll_add(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, "21", 2); + fieldstat_hll_add(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, "22", 2); + fieldstat_hll_add(instance, cube_id, metric_id, &TEST_FIELD_DOUBLE, 1, "31", 2); + fieldstat_hll_add(instance, cube_id, metric_id, &TEST_FIELD_DOUBLE, 1, "32", 2); + fieldstat_hll_add(instance, cube_id, metric_id, &TEST_FIELD_DOUBLE, 1, "33", 2); + fieldstat_merge(instance_dest, instance); + + struct field_list *tag_list = NULL; + size_t n_cell = 0; + fieldstat_cube_get_cells(instance_dest, 0, &tag_list, &n_cell); + EXPECT_EQ(n_cell, 2); + EXPECT_NEAR(merge_test_fieldstat_hll_get(instance_dest, 0, 0, &tag_list[0]), 3, 0.3); + EXPECT_NEAR(merge_test_fieldstat_hll_get(instance_dest, 0, 0, &tag_list[1]), 2, 0.3); + EXPECT_STREQ(tag_list[0].field[0].key, TEST_FIELD_DOUBLE.key); + EXPECT_STREQ(tag_list[1].field[0].key, TEST_FIELD_INT.key); + + fieldstat_free(instance); + fieldstat_free(instance_dest); + fieldstat_field_list_arr_free(tag_list, n_cell); +} + + +TEST(unit_test_merge, gen_dest_full_all_src_inserted_given_src_flows_larger_spreadsketch) { + int K = 100; + SpreadSketchZipfGenerator flow_generator(1.0, K); // exactly the number of cells, so there will be almost all(in case of hash collision happen) cells added successfully + struct fieldstat *instance_src = fieldstat_new(); + int cube_id = fieldstat_cube_create(instance_src, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_hll(instance_src, cube_id, "metric", 6); + fieldstat_cube_set_sampling(instance_src, cube_id, SAMPLING_MODE_TOP_CARDINALITY, K, 0); + struct fieldstat *instance_dest = fieldstat_fork(instance_src); + const char dest_key[] = "key of dest"; + const char src_key[] = "key of src"; + + std::unordered_map<std::string, std::unordered_set<std::string>> flow_cnt; + for (int i = 0; i < 500000; i++) { // add more, so the fanout of any flow to src instance is more than dest + Flow flow = flow_generator.next(); + Fieldstat_tag_list_wrapper dimension = Fieldstat_tag_list_wrapper(src_key, flow.src_ip.c_str()); + Fieldstat_tag_list_wrapper item = Fieldstat_tag_list_wrapper("dummy", flow.dst_ip.c_str()); + fieldstat_hll_add_field(instance_src, cube_id, metric_id, dimension.get_tag(), dimension.get_tag_count(), item.get_tag(), item.get_tag_count()); + + flow_cnt[dimension.to_string()].insert(item.to_string()); + } + + for (int i = 0; i < 1000; i++) { + Flow flow = flow_generator.next(); + Fieldstat_tag_list_wrapper dimension = Fieldstat_tag_list_wrapper(dest_key, flow.src_ip.c_str()); + Fieldstat_tag_list_wrapper item = Fieldstat_tag_list_wrapper("dummy", flow.dst_ip.c_str()); + fieldstat_hll_add_field(instance_dest, cube_id, metric_id, dimension.get_tag(), dimension.get_tag_count(), item.get_tag(), item.get_tag_count()); + + flow_cnt[dimension.to_string()].insert(item.to_string()); + } + + fieldstat_merge(instance_dest, instance_src); + + struct field_list *tag_list = NULL; + struct field_list *tag_list_src = NULL; + size_t n_cell = 0; + size_t n_cell_src = 0; + std::vector<struct Fieldstat_tag_list_wrapper *> test_result; + fieldstat_cube_get_cells(instance_dest, 0, &tag_list, &n_cell); + fieldstat_cube_get_cells(instance_src, 0, &tag_list_src, &n_cell_src); + for (size_t i = 0; i < n_cell; i++) { + test_result.push_back(new Fieldstat_tag_list_wrapper(&tag_list[i])); + } + std::unordered_map<std::string, int> expected_unique_cnt; + for (auto &kv : flow_cnt) { + expected_unique_cnt[kv.first] = kv.second.size(); + } + + double recall = test_cal_topk_accuracy(test_result, expected_unique_cnt); + EXPECT_NEAR(recall, n_cell_src * 1.0 / n_cell, 0.0001); // the false positive is only generated because some cells in src are left because of hash collision + + fieldstat_free(instance_src); + fieldstat_free(instance_dest); + fieldstat_field_list_arr_free(tag_list, n_cell); + fieldstat_field_list_arr_free(tag_list_src, n_cell_src); + for (size_t i = 0; i < test_result.size(); i++) { + delete test_result[i]; + } +} + +TEST(unit_test_merge, merge_accuracy_test_gen_dest_full_some_inserted_and_some_merged_and_some_fail_to_add_spreadsketch) { + int K = 10; + SpreadSketchZipfGenerator flow_generator(1.0, K * 10); + struct fieldstat *instance_src = fieldstat_new(); + int cube_id = fieldstat_cube_create(instance_src, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_hll(instance_src, cube_id, "metric", 6); + fieldstat_cube_set_sampling(instance_src, cube_id, SAMPLING_MODE_TOP_CARDINALITY, K, 0); + struct fieldstat *instance_dest = fieldstat_fork(instance_src); + + std::unordered_map<std::string, std::unordered_set<std::string>> flow_cnt; + for (int i = 0; i < 100000; i++) { + Flow flow = flow_generator.next(); + const char *use_key = rand()%2? "src":"common"; + Fieldstat_tag_list_wrapper dimension = Fieldstat_tag_list_wrapper(use_key, flow.src_ip.c_str()); + Fieldstat_tag_list_wrapper item = Fieldstat_tag_list_wrapper("dummy", flow.dst_ip.c_str()); + fieldstat_hll_add_field(instance_src, cube_id, metric_id, dimension.get_tag(), dimension.get_tag_count(), item.get_tag(), item.get_tag_count()); + + flow_cnt[dimension.to_string()].insert(item.to_string()); + } + for (int i = 0; i < 100000; i++) { + Flow flow = flow_generator.next(); + const char *use_key = rand()%2? "dest":"common"; + Fieldstat_tag_list_wrapper dimension = Fieldstat_tag_list_wrapper(use_key, flow.src_ip.c_str()); + Fieldstat_tag_list_wrapper item = Fieldstat_tag_list_wrapper("dummy", flow.dst_ip.c_str()); + fieldstat_hll_add_field(instance_src, cube_id, metric_id, dimension.get_tag(), dimension.get_tag_count(), item.get_tag(), item.get_tag_count()); + + flow_cnt[dimension.to_string()].insert(item.to_string()); + } + + fieldstat_merge(instance_dest, instance_src); + + struct field_list *tag_list = NULL; + size_t n_cell = 0; + std::vector<struct Fieldstat_tag_list_wrapper *> test_result; + fieldstat_cube_get_cells(instance_dest, 0, &tag_list, &n_cell); + for (size_t i = 0; i < n_cell; i++) { + test_result.push_back(new Fieldstat_tag_list_wrapper(&tag_list[i])); + } + + std::unordered_map<std::string, int> expected_unique_cnt; + for (auto &kv : flow_cnt) { + expected_unique_cnt[kv.first] = kv.second.size(); + } + double recall = test_cal_topk_accuracy(test_result, expected_unique_cnt); + EXPECT_GE(recall, 0.7); + printf("gen_dest_full_all_src_inserted_given_src_flows_larger_spreadsketch recall is %lf\n", recall); + + fieldstat_free(instance_src); + fieldstat_free(instance_dest); + fieldstat_field_list_arr_free(tag_list, n_cell); + for (size_t i = 0; i < test_result.size(); i++) { + delete test_result[i]; + } +} int main(int argc, char *argv[]) { testing::InitGoogleTest(&argc, argv); + // testing::GTEST_FLAG(filter) = "*gen_dest_full_all_src_inserted_given_src_flows_larger_spreadsketch"; + return RUN_ALL_TESTS(); }
\ No newline at end of file diff --git a/test/test_metric_counter.cpp b/test/test_metric_counter.cpp index de0f42b..98b52dd 100644 --- a/test/test_metric_counter.cpp +++ b/test/test_metric_counter.cpp @@ -7,22 +7,33 @@ using namespace std; + struct fieldstat *test_init_standard_instance() { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); EXPECT_EQ(cube_id, 0); - int metric_id = fieldstat_register_counter(instance, "czz_test counter metric"); + int metric_id = fieldstat_register_counter(instance, cube_id, "czz_test counter metric"); EXPECT_EQ(metric_id, 0); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_COMPREHENSIVE, 10, 0); + return instance; } -long long my_fieldstat_counter_get(const struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag_list *tag_list = &TEST_TAG_LIST_INT) +long long my_fieldstat_counter_get(const struct fieldstat *instance, int cube_id, int metric_id, const struct field_list *tag_list = &TEST_FIELD_LIST_INT) { long long ret = 0; - fieldstat_counter_get(instance, cube_id, metric_id, tag_list, &ret); + fieldstat_counter_get(instance, cube_id, tag_list, metric_id, &ret); + return ret; +} + +int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k, int primary_metric_id=0) +{ + assert(mode == SAMPLING_MODE_COMPREHENSIVE); + int ret = fieldstat_cube_create(instance, dimensions, n_dimensions); + fieldstat_cube_set_sampling(instance, ret, mode, k, primary_metric_id); return ret; } @@ -35,25 +46,32 @@ void test_assert_standard_instance(const struct fieldstat *instance) int ret_cell_id = ret_cube_id_arr[0]; free(ret_cube_id_arr); EXPECT_EQ(ret_cell_id, 0); - const char *name = fieldstat_get_metric_name(instance, 0); + const char *name = fieldstat_get_metric_name(instance, ret_cell_id, 0); EXPECT_STREQ(name, "czz_test counter metric"); - struct fieldstat_tag_list *tag_list = NULL; + struct field_list *tag_list = NULL; size_t n_cell = 0; - fieldstat_get_cells_used_by_metric(instance, 0, 0, &tag_list, &n_cell); + fieldstat_cube_get_cells(instance, 0, &tag_list, &n_cell); EXPECT_EQ(n_cell, 1); - EXPECT_EQ(tag_list->n_tag, 1); - EXPECT_STREQ(tag_list->tag[0].key, TEST_TAG_INT.key); + EXPECT_EQ(tag_list->n_field, 1); + EXPECT_STREQ(tag_list->field[0].key, TEST_FIELD_INT.key); - fieldstat_tag_list_arr_free(tag_list, n_cell); + int *metric_id = NULL; + size_t n_metric = 0; + fieldstat_get_metric_in_cell(instance, 0, &tag_list[0], &metric_id, &n_metric); + EXPECT_EQ(n_metric, 1); + EXPECT_EQ(metric_id[0], 0); + free(metric_id); + + fieldstat_field_list_arr_free(tag_list, n_cell); } TEST(metric_test_counter, simple_register_and_query_counter) { struct fieldstat *instance = test_init_standard_instance(); - fieldstat_counter_incrby(instance, 0, 0, &TEST_TAG_INT, 1, 10000); - fieldstat_counter_incrby(instance, 0, 0, &TEST_TAG_INT, 1, 86); + fieldstat_counter_incrby(instance, 0, 0, &TEST_FIELD_INT, 1, 10000); + fieldstat_counter_incrby(instance, 0, 0, &TEST_FIELD_INT, 1, 86); test_assert_standard_instance(instance); EXPECT_EQ(my_fieldstat_counter_get(instance, 0, 0), 10086); @@ -64,8 +82,8 @@ TEST(metric_test_counter, simple_register_and_query_counter) TEST(metric_test_counter, merge_counter) { struct fieldstat *instance = test_init_standard_instance(); - fieldstat_counter_incrby(instance, 0, 0, &TEST_TAG_INT, 1, 100); - fieldstat_counter_incrby(instance, 0, 0, &TEST_TAG_INT, 1, -10000); + fieldstat_counter_incrby(instance, 0, 0, &TEST_FIELD_INT, 1, 100); + fieldstat_counter_incrby(instance, 0, 0, &TEST_FIELD_INT, 1, -10000); struct fieldstat *instance_total = fieldstat_new(); fieldstat_merge(instance_total, instance); @@ -80,16 +98,16 @@ TEST(metric_test_counter, merge_counter) fieldstat_free(instance_total); } -TEST(metric_test_counter, serialization_and_merge_counter_twice_with_reset) +TEST(metric_test_counter, merge_counter_twice_with_reset) { struct fieldstat *instance = test_init_standard_instance(); - fieldstat_counter_incrby(instance, 0, 0, &TEST_TAG_INT, 1, 10086); + fieldstat_counter_incrby(instance, 0, 0, &TEST_FIELD_INT, 1, 10086); struct fieldstat *instance_total = fieldstat_new(); fieldstat_merge(instance_total, instance); fieldstat_reset(instance); - fieldstat_counter_incrby(instance, 0, 0, &TEST_TAG_INT, 1, 4); + fieldstat_counter_incrby(instance, 0, 0, &TEST_FIELD_INT, 1, 4); fieldstat_merge(instance_total, instance); test_assert_standard_instance(instance_total); @@ -103,11 +121,12 @@ TEST(metric_test_counter, serialization_and_merge_counter_twice_with_reset) TEST(metric_test_counter, topk_add_and_test_accuracy) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_TOPK, 10); - fieldstat_register_counter(instance, "test"); + fieldstat_cube_create(instance, &TEST_FIELD_INT_2, 1); + fieldstat_register_counter(instance, 0, "test"); + fieldstat_cube_set_sampling(instance, 0, SAMPLING_MODE_TOPK, 10, 0); int tag_list_num = 10000; - Fieldstat_tag_list_wrapper *tags[tag_list_num]; + Fieldstat_tag_list_wrapper *fields[tag_list_num]; map<string, int> flow_cnt; for (int i = 0; i < tag_list_num; i++) { @@ -120,18 +139,18 @@ TEST(metric_test_counter, topk_add_and_test_accuracy) flow_k = "mouse"; flow_v = std::to_string(rand() % 1000); } - tags[i] = new Fieldstat_tag_list_wrapper(flow_k, flow_v.c_str()); + fields[i] = new Fieldstat_tag_list_wrapper(flow_k, flow_v.c_str()); flow_cnt[flow_k + flow_v]++; } for (int i = 0; i < tag_list_num; i++) { - fieldstat_counter_incrby(instance, 0, 0, tags[i]->get_tag(), tags[i]->get_tag_count(), 1); + fieldstat_counter_incrby(instance, 0, 0, fields[i]->get_tag(), fields[i]->get_tag_count(), 1); } - struct fieldstat_tag_list *tag_list = NULL; + struct field_list *tag_list = NULL; size_t n_cell = 0; - fieldstat_get_cells_used_by_metric(instance, 0, 0, &tag_list, &n_cell); + fieldstat_cube_get_cells(instance, 0, &tag_list, &n_cell); EXPECT_EQ(n_cell, 10); long long error = 0; for (size_t i = 0; i < n_cell; i++) { @@ -148,49 +167,21 @@ TEST(metric_test_counter, topk_add_and_test_accuracy) printf("topk_add_and_test_accuracy Mean ratio e: %lld\n", error); EXPECT_LT(error, tag_list_num * 0.005); - fieldstat_tag_list_arr_free(tag_list, n_cell); + fieldstat_field_list_arr_free(tag_list, n_cell); fieldstat_free(instance); for (int i = 0; i < tag_list_num; i++) - delete tags[i]; -} - -TEST(metric_test_counter, topk_add_with_monotonically_increasing_flow_expecting_add_fail) -{ - struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_TOPK, 10); - fieldstat_register_counter(instance, "test"); - - const int end_cell_id = 10 * 8; - const int tag_list_num = 100; - Fieldstat_tag_list_wrapper *tags[tag_list_num]; - for (int i = 0; i < tag_list_num; i++) - { - tags[i] = new Fieldstat_tag_list_wrapper("elephant", std::to_string(i).c_str()); - } - - for (int i = 0; i < tag_list_num; i++) { - int ret = fieldstat_counter_incrby(instance, 0, 0, tags[i]->get_tag(), tags[i]->get_tag_count(), i + 1); - if (ret < 0) { - EXPECT_EQ(i, end_cell_id); - EXPECT_EQ(ret, FS_ERR_TOO_MANY_CELLS); - break; - } - } - - fieldstat_free(instance); - for (int i = 0; i < tag_list_num; i++) - delete tags[i]; + delete fields[i]; } TEST(metric_test_counter, add_with_wrong_cube_id_expecting_fail) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_TOPK, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_FIELD_INT_2, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int ret = fieldstat_counter_incrby(instance, cube_id + 1, 0, &TEST_TAG_INT, 1, 1); + int ret = fieldstat_counter_incrby(instance, cube_id + 1, 0, &TEST_FIELD_INT, 1, 1); EXPECT_EQ(ret, FS_ERR_INVALID_CUBE_ID); - ret = fieldstat_counter_incrby(instance, -1, 0, &TEST_TAG_INT, 1, 1); + ret = fieldstat_counter_incrby(instance, -1, 0, &TEST_FIELD_INT, 1, 1); EXPECT_EQ(ret, FS_ERR_INVALID_CUBE_ID); fieldstat_free(instance); @@ -199,12 +190,12 @@ TEST(metric_test_counter, add_with_wrong_cube_id_expecting_fail) TEST(metric_test_counter, add_with_wrong_metric_id_expecting_fail) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_TOPK, 10); - int metric_id = fieldstat_register_counter(instance, "test"); + int cube_id = test_fieldstat_cube_create(instance, &TEST_FIELD_INT_2, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int metric_id = fieldstat_register_counter(instance, 0, "test"); - int ret = fieldstat_counter_incrby(instance, cube_id, metric_id + 1, &TEST_TAG_INT, 1, 1); + int ret = fieldstat_counter_incrby(instance, cube_id, metric_id + 1, &TEST_FIELD_INT, 1, 1); EXPECT_EQ(ret, FS_ERR_INVALID_METRIC_ID); - ret = fieldstat_counter_incrby(instance, cube_id, -1, &TEST_TAG_INT, 1, 1); + ret = fieldstat_counter_incrby(instance, cube_id, -1, &TEST_FIELD_INT, 1, 1); EXPECT_EQ(ret, FS_ERR_INVALID_METRIC_ID); fieldstat_free(instance); @@ -213,57 +204,16 @@ TEST(metric_test_counter, add_with_wrong_metric_id_expecting_fail) TEST(metric_test_counter, add_and_query_on_dummy_cell_of_topk) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); - fieldstat_register_counter(instance, "primary"); // also the dummy one - int metric_id = fieldstat_register_counter(instance, "using"); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + fieldstat_register_counter(instance, cube_id, "primary"); // also the dummy one + int metric_id = fieldstat_register_counter(instance, cube_id, "using"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, 0); - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, 1); // add success long long measure = my_fieldstat_counter_get(instance, cube_id, metric_id); EXPECT_EQ(measure, 1); - // cannot query dummy - int *metric_ids = NULL; - size_t n_metric = 0; - fieldstat_get_metrics_used_by_cube(instance, cube_id, &metric_ids, &n_metric); - EXPECT_EQ(n_metric, 1); - EXPECT_EQ(metric_ids[0], 1); - free(metric_ids); - - struct fieldstat_tag_list *tag_list = NULL; - size_t n_cell; - fieldstat_get_cells_used_by_metric(instance, cube_id, 0, &tag_list, &n_cell); - EXPECT_EQ(n_cell, 0); - - fieldstat_free(instance); -} - -TEST(metric_test_counter, set_on_primary_metric_going_smaller) -{ - struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); - int metric_id = fieldstat_register_counter(instance, "primary"); - - int ret = fieldstat_counter_set(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 10); - EXPECT_EQ(ret, FS_OK); - ret = fieldstat_counter_set(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 2); - EXPECT_EQ(ret, FS_ERR_INVALID_PARAM); - - fieldstat_free(instance); -} - -TEST(metric_test_counter, set_on_primary_metric_going_bigger) -{ - struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); - int metric_id = fieldstat_register_counter(instance, "primary"); - - int ret = fieldstat_counter_set(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 10); - EXPECT_EQ(ret, FS_OK); - ret = fieldstat_counter_set(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 20); - EXPECT_EQ(ret, FS_OK); - - EXPECT_EQ(my_fieldstat_counter_get(instance, cube_id, metric_id), 20); fieldstat_free(instance); } @@ -271,68 +221,34 @@ TEST(metric_test_counter, set_on_primary_metric_going_bigger) TEST(metric_test_counter, primary_counter_add_after_first) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); - int metric_id_primary = fieldstat_register_counter(instance, "primary"); - int metric_id2 = fieldstat_register_counter(instance, "using"); - fieldstat_cube_set_primary_metric(instance, cube_id, metric_id_primary); - + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id_primary = fieldstat_register_counter(instance, cube_id, "primary"); + int metric_id2 = fieldstat_register_counter(instance, cube_id, "using"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, metric_id_primary); - int ret = fieldstat_counter_incrby(instance, cube_id, metric_id2, &TEST_TAG_INT, 1, 10); + int ret = fieldstat_counter_incrby(instance, cube_id, metric_id2, &TEST_FIELD_INT, 1, 10); EXPECT_EQ(ret, FS_OK); - ret = fieldstat_counter_incrby(instance, cube_id, metric_id_primary, &TEST_TAG_INT, 1, 20); + ret = fieldstat_counter_incrby(instance, cube_id, metric_id_primary, &TEST_FIELD_INT, 1, 20); EXPECT_EQ(ret, FS_OK); EXPECT_EQ(my_fieldstat_counter_get(instance, cube_id, metric_id_primary), 20); EXPECT_EQ(my_fieldstat_counter_get(instance, cube_id, metric_id2), 10); // TOPK functions well - struct fieldstat_tag tag = TEST_TAG_INT; + struct field field = TEST_FIELD_INT; for (int i = 0; i < 9; i++) { - tag.value_longlong = i + 123; - ret = fieldstat_counter_incrby(instance, cube_id, metric_id_primary, &tag, 1, 1 + i); + field.value_longlong = i + 123; + ret = fieldstat_counter_incrby(instance, cube_id, metric_id_primary, &field, 1, 1 + i); EXPECT_EQ(ret, FS_OK); } // now the TOPK is full, add a new one - tag.value_longlong = 321; - ret = fieldstat_counter_incrby(instance, cube_id, metric_id_primary, &tag, 1, 1); + field.value_longlong = 321; + ret = fieldstat_counter_incrby(instance, cube_id, metric_id_primary, &field, 1, 1); EXPECT_EQ(ret, FS_ERR_TOO_MANY_CELLS); fieldstat_free(instance); } -TEST(metric_test_counter, topk_set_and_test_accuracy) -{ - struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_TOPK, 6); - fieldstat_register_counter(instance, "test"); - - struct fieldstat_tag tag = TEST_TAG_INT; - // tag : [0, 1, 2, 3, 4 ,5] - // value: [0, 1, 2, 3, 4 ,5] - for (int i = 0; i < 6; i++) { - tag.value_longlong = i; - EXPECT_EQ(fieldstat_counter_set(instance, 0, 0, &tag, 1, i), FS_OK); - } - // tag : [0, 1, 2, 3, 4 ,5, 6, 7, 8] - // value: [0, 1, 2, 100, 100, 100, 100, 100, 100] - for (int i = 0; i < 6; i++) { - tag.value_longlong = i + 3; - EXPECT_EQ(fieldstat_counter_set(instance, 0, 0, &tag, 1, 100), FS_OK); - } - - struct fieldstat_tag_list *tag_list = NULL; - size_t n_cell = 0; - fieldstat_get_cells_used_by_metric(instance, 0, 0, &tag_list, &n_cell); - EXPECT_EQ(n_cell, 6); - for (size_t i = 0; i < n_cell; i++) { - EXPECT_EQ(tag_list[i].tag[0].value_longlong, i + 3); - EXPECT_EQ(my_fieldstat_counter_get(instance, 0, 0, &tag_list[i]), 100); - } - - fieldstat_tag_list_arr_free(tag_list, n_cell); - fieldstat_free(instance); -} - int main(int argc, char *argv[]) { testing::InitGoogleTest(&argc, argv); diff --git a/test/test_metric_histogram.cpp b/test/test_metric_histogram.cpp index 6983d02..c99a565 100644 --- a/test/test_metric_histogram.cpp +++ b/test/test_metric_histogram.cpp @@ -6,13 +6,21 @@ #include "hdr/hdr_histogram.h" #include "histogram_encoder.h" +int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k) +{ + assert(mode == SAMPLING_MODE_COMPREHENSIVE); + int ret = fieldstat_cube_create(instance, dimensions, n_dimensions); + fieldstat_cube_set_sampling(instance, ret, mode, k, 0); + return ret; +} + struct fieldstat *test_init_standard_instance_one_cube_one_metric_one_cell_hdr() { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); EXPECT_EQ(cube_id, 0); - int metric_id = fieldstat_register_hist(instance, "czz_test hdr metric", 1, 600000, 3); + int metric_id = fieldstat_register_histogram(instance, cube_id, "czz_test hdr metric", 1, 600000, 3); EXPECT_EQ(metric_id, 0); return instance; @@ -27,31 +35,38 @@ void test_assert_standard_instance(const struct fieldstat *instance) int ret_cell_id = ret_cube_id_arr[0]; free(ret_cube_id_arr); EXPECT_EQ(ret_cell_id, 0); - const char *name = fieldstat_get_metric_name(instance, 0); + const char *name = fieldstat_get_metric_name(instance, ret_cell_id, 0); EXPECT_STREQ(name, "czz_test hdr metric"); - struct fieldstat_tag_list *tag_list = NULL; + struct field_list *tag_list = NULL; size_t n_cell = 0; - fieldstat_get_cells_used_by_metric(instance, 0, 0, &tag_list, &n_cell); + fieldstat_cube_get_cells(instance, 0, &tag_list, &n_cell); EXPECT_EQ(n_cell, 1); - EXPECT_EQ(tag_list->n_tag, 1); - EXPECT_STREQ(tag_list->tag[0].key, TEST_TAG_STRING.key); + EXPECT_EQ(tag_list->n_field, 1); + EXPECT_STREQ(tag_list->field[0].key, TEST_FIELD_STRING.key); + + int *metric_id = NULL; + size_t n_metric = 0; + fieldstat_get_metric_in_cell(instance, 0, &tag_list[0], &metric_id, &n_metric); + EXPECT_EQ(n_metric, 1); + EXPECT_EQ(metric_id[0], 0); + free(metric_id); - fieldstat_tag_list_arr_free(tag_list, n_cell); + fieldstat_field_list_arr_free(tag_list, n_cell); } TEST(metric_test_histogram, simple_register_and_query) { struct fieldstat *instance = test_init_standard_instance_one_cube_one_metric_one_cell_hdr(); - fieldstat_hist_record(instance, 0, 0, &TEST_TAG_STRING, 1, 1234); - fieldstat_hist_record(instance, 0, 0, &TEST_TAG_STRING, 1, 1234); - fieldstat_hist_record(instance, 0, 0, &TEST_TAG_STRING, 1, 123); + fieldstat_histogram_record(instance, 0, 0, &TEST_FIELD_STRING, 1, 1234); + fieldstat_histogram_record(instance, 0, 0, &TEST_FIELD_STRING, 1, 1234); + fieldstat_histogram_record(instance, 0, 0, &TEST_FIELD_STRING, 1, 123); test_assert_standard_instance(instance); - EXPECT_EQ(fieldstat_hist_value_at_percentile(instance, 0, 0, &TEST_TAG_LIST_STRING, 50.0), 1234); - EXPECT_EQ(fieldstat_hist_count_le_value(instance, 0, 0, &TEST_TAG_LIST_STRING, 1000), 1); + EXPECT_EQ(fieldstat_histogram_value_at_percentile(instance, 0, &TEST_FIELD_LIST_STRING, 0, 50.0), 1234); + EXPECT_EQ(fieldstat_histogram_count_le_value(instance, 0, &TEST_FIELD_LIST_STRING, 0, 1000), 1); fieldstat_free(instance); } @@ -59,9 +74,9 @@ TEST(metric_test_histogram, simple_register_and_query) TEST(metric_test_histogram, merge) { struct fieldstat *instance = test_init_standard_instance_one_cube_one_metric_one_cell_hdr(); - fieldstat_hist_record(instance, 0, 0, &TEST_TAG_STRING, 1, 1234); - fieldstat_hist_record(instance, 0, 0, &TEST_TAG_STRING, 1, 1234); - fieldstat_hist_record(instance, 0, 0, &TEST_TAG_STRING, 1, 123); + fieldstat_histogram_record(instance, 0, 0, &TEST_FIELD_STRING, 1, 1234); + fieldstat_histogram_record(instance, 0, 0, &TEST_FIELD_STRING, 1, 1234); + fieldstat_histogram_record(instance, 0, 0, &TEST_FIELD_STRING, 1, 123); struct fieldstat *instance_total = fieldstat_new(); fieldstat_merge(instance_total, instance); @@ -69,8 +84,8 @@ TEST(metric_test_histogram, merge) // query test_assert_standard_instance(instance_total); - EXPECT_EQ(fieldstat_hist_value_at_percentile(instance_total, 0, 0, &TEST_TAG_LIST_STRING, 50.0), 1234); - EXPECT_EQ(fieldstat_hist_count_le_value(instance, 0, 0, &TEST_TAG_LIST_STRING, 1000), 1); + EXPECT_EQ(fieldstat_histogram_value_at_percentile(instance_total, 0, &TEST_FIELD_LIST_STRING, 0, 50.0), 1234); + EXPECT_EQ(fieldstat_histogram_count_le_value(instance, 0, &TEST_FIELD_LIST_STRING, 0, 1000), 1); fieldstat_free(instance); fieldstat_free(instance_total); @@ -79,19 +94,19 @@ TEST(metric_test_histogram, merge) TEST(metric_test_histogram, merge_twice_with_reset) { struct fieldstat *instance = test_init_standard_instance_one_cube_one_metric_one_cell_hdr(); - fieldstat_hist_record(instance, 0, 0, &TEST_TAG_STRING, 1, 1234); - fieldstat_hist_record(instance, 0, 0, &TEST_TAG_STRING, 1, 123); + fieldstat_histogram_record(instance, 0, 0, &TEST_FIELD_STRING, 1, 1234); + fieldstat_histogram_record(instance, 0, 0, &TEST_FIELD_STRING, 1, 123); struct fieldstat *instance_total = fieldstat_new(); fieldstat_merge(instance_total, instance); fieldstat_reset(instance); - fieldstat_hist_record(instance, 0, 0, &TEST_TAG_STRING, 1, 1234); + fieldstat_histogram_record(instance, 0, 0, &TEST_FIELD_STRING, 1, 1234); fieldstat_merge(instance_total, instance); test_assert_standard_instance(instance_total); - EXPECT_EQ(fieldstat_hist_value_at_percentile(instance_total, 0, 0, &TEST_TAG_LIST_STRING, 50.0), 1234); - EXPECT_EQ(fieldstat_hist_count_le_value(instance_total, 0, 0, &TEST_TAG_LIST_STRING, 1000), 1); + EXPECT_EQ(fieldstat_histogram_value_at_percentile(instance_total, 0, &TEST_FIELD_LIST_STRING, 0, 50.0), 1234); + EXPECT_EQ(fieldstat_histogram_count_le_value(instance_total, 0, &TEST_FIELD_LIST_STRING, 0, 1000), 1); fieldstat_free(instance); fieldstat_free(instance_total); @@ -100,11 +115,11 @@ TEST(metric_test_histogram, merge_twice_with_reset) TEST(metric_test_histogram, add_with_wrong_cube_id_expecting_fail) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_FIELD_INT_2, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int ret = fieldstat_hist_record(instance, cube_id + 1, 0, &TEST_TAG_INT, 1, 1); + int ret = fieldstat_histogram_record(instance, cube_id + 1, 0, &TEST_FIELD_INT, 1, 1); EXPECT_EQ(ret, FS_ERR_INVALID_CUBE_ID); - ret = fieldstat_hist_record(instance, -1, 0, &TEST_TAG_INT, 1, 1); + ret = fieldstat_histogram_record(instance, -1, 0, &TEST_FIELD_INT, 1, 1); EXPECT_EQ(ret, FS_ERR_INVALID_CUBE_ID); fieldstat_free(instance); @@ -113,12 +128,12 @@ TEST(metric_test_histogram, add_with_wrong_cube_id_expecting_fail) TEST(metric_test_histogram, add_with_wrong_metric_id_expecting_fail) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int metric_id = fieldstat_register_hist(instance, "czz_test", 1, 600000, 3); + int cube_id = test_fieldstat_cube_create(instance, &TEST_FIELD_INT_2, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int metric_id = fieldstat_register_histogram(instance, cube_id, "czz_test", 1, 600000, 3); - int ret = fieldstat_hist_record(instance, cube_id, metric_id + 1, &TEST_TAG_INT, 1, 1); + int ret = fieldstat_histogram_record(instance, cube_id, metric_id + 1, &TEST_FIELD_INT, 1, 1); EXPECT_EQ(ret, FS_ERR_INVALID_METRIC_ID); - ret = fieldstat_hist_record(instance, cube_id, -1, &TEST_TAG_INT, 1, 1); + ret = fieldstat_histogram_record(instance, cube_id, -1, &TEST_FIELD_INT, 1, 1); EXPECT_EQ(ret, FS_ERR_INVALID_METRIC_ID); fieldstat_free(instance); @@ -162,16 +177,16 @@ TEST(metric_test_histogram, encode_decode_b64) TEST(metric_test_histogram, can_add_0value) // histogram only allow min_val > 0, but it can accept value == 0 { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int metric_id = fieldstat_register_hist(instance, "czz_test", 1, 600000, 3); + int cube_id = test_fieldstat_cube_create(instance, &TEST_FIELD_INT_2, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int metric_id = fieldstat_register_histogram(instance, cube_id, "czz_test", 1, 600000, 3); - int ret = fieldstat_hist_record(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 0); + int ret = fieldstat_histogram_record(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, 0); EXPECT_EQ(ret, 0); - ret = fieldstat_hist_record(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 0); + ret = fieldstat_histogram_record(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, 0); EXPECT_EQ(ret, 0); - ret = fieldstat_hist_record(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 500); + ret = fieldstat_histogram_record(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, 500); - EXPECT_EQ(fieldstat_hist_value_at_percentile(instance, cube_id, metric_id, &TEST_TAG_LIST_INT, 60.0), 0); + EXPECT_EQ(fieldstat_histogram_value_at_percentile(instance, cube_id, &TEST_FIELD_LIST_INT, metric_id, 60.0), 0); fieldstat_free(instance); } diff --git a/test/test_metric_hll.cpp b/test/test_metric_hll.cpp index 4f7f9d2..bbfc36a 100644 --- a/test/test_metric_hll.cpp +++ b/test/test_metric_hll.cpp @@ -1,14 +1,28 @@ #include <gtest/gtest.h> +#include <string> +#include <vector> +#include <unordered_map> +#include <unordered_set> +#include <math.h> + #include "fieldstat.h" #include "utils.hpp" +int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k) +{ + assert(mode == SAMPLING_MODE_COMPREHENSIVE); + int ret = fieldstat_cube_create(instance, dimensions, n_dimensions); + fieldstat_cube_set_sampling(instance, ret, mode, k, 0); + return ret; +} + struct fieldstat *test_init_standard_instance_one_cube_one_metric_one_cell_hll(bool is_gauge = false) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); EXPECT_EQ(cube_id, 0); - int metric_id = fieldstat_register_hll(instance, "czz_test hll metric", 10); + int metric_id = fieldstat_register_hll(instance, cube_id, "czz_test hll metric", 10); EXPECT_EQ(metric_id, 0); return instance; @@ -23,23 +37,30 @@ void test_assert_standard_instance(const struct fieldstat *instance) int ret_cell_id = ret_cube_id_arr[0]; free(ret_cube_id_arr); EXPECT_EQ(ret_cell_id, 0); - const char *name = fieldstat_get_metric_name(instance, 0); + const char *name = fieldstat_get_metric_name(instance, ret_cell_id, 0); EXPECT_STREQ(name, "czz_test hll metric"); - struct fieldstat_tag_list *tag_list = NULL; + struct field_list *tag_list = NULL; size_t n_cell = 0; - fieldstat_get_cells_used_by_metric(instance, 0, 0, &tag_list, &n_cell); + fieldstat_cube_get_cells(instance, 0, &tag_list, &n_cell); EXPECT_EQ(n_cell, 1); - EXPECT_EQ(tag_list->n_tag, 1); - EXPECT_STREQ(tag_list->tag[0].key, TEST_TAG_INT.key); - - fieldstat_tag_list_arr_free(tag_list, n_cell); + EXPECT_EQ(tag_list->n_field, 1); + EXPECT_STREQ(tag_list->field[0].key, TEST_FIELD_INT.key); + + int *metric_id = NULL; + size_t n_metric = 0; + fieldstat_get_metric_in_cell(instance, 0, &tag_list[0], &metric_id, &n_metric); + EXPECT_EQ(n_metric, 1); + EXPECT_EQ(metric_id[0], 0); + free(metric_id); + + fieldstat_field_list_arr_free(tag_list, n_cell); } double my_fieldstat_hll_get(const struct fieldstat *instance, int cube_id, int metric_id) { double ret = 0; - fieldstat_hll_get(instance, cube_id, metric_id, &TEST_TAG_LIST_INT, &ret); + fieldstat_hll_get(instance, cube_id, &TEST_FIELD_LIST_INT, metric_id, &ret); return ret; } @@ -47,10 +68,23 @@ TEST(metric_test_hll, simple_register_and_query) { struct fieldstat *instance = test_init_standard_instance_one_cube_one_metric_one_cell_hll(); - fieldstat_hll_add(instance, 0, 0, &TEST_TAG_INT, 1, "hello", 5); - fieldstat_hll_add(instance, 0, 0, &TEST_TAG_INT, 1, "wor", 3); - fieldstat_hll_add(instance, 0, 0, &TEST_TAG_INT, 1, "world", 5); + fieldstat_hll_add(instance, 0, 0, &TEST_FIELD_INT, 1, "hello", 5); + fieldstat_hll_add(instance, 0, 0, &TEST_FIELD_INT, 1, "wor", 3); + fieldstat_hll_add(instance, 0, 0, &TEST_FIELD_INT, 1, "world", 5); + + test_assert_standard_instance(instance); + EXPECT_NEAR(my_fieldstat_hll_get(instance, 0, 0), 3, 0.5); + + fieldstat_free(instance); +} + +TEST(metric_test_hll, add_with_tags) { + struct fieldstat *instance = test_init_standard_instance_one_cube_one_metric_one_cell_hll(); + fieldstat_hll_add_field(instance, 0, 0, &TEST_FIELD_INT, 1, &TEST_FIELD_INT, 1); + fieldstat_hll_add_field(instance, 0, 0, &TEST_FIELD_INT, 1, &TEST_FIELD_DOUBLE, 1); + fieldstat_hll_add_field(instance, 0, 0, &TEST_FIELD_INT, 1, &TEST_FIELD_STRING, 1); + test_assert_standard_instance(instance); EXPECT_NEAR(my_fieldstat_hll_get(instance, 0, 0), 3, 0.5); @@ -60,9 +94,9 @@ TEST(metric_test_hll, simple_register_and_query) TEST(metric_test_hll, merge) { struct fieldstat *instance = test_init_standard_instance_one_cube_one_metric_one_cell_hll(); - fieldstat_hll_add(instance, 0, 0, &TEST_TAG_INT, 1, "hello", 5); - fieldstat_hll_add(instance, 0, 0, &TEST_TAG_INT, 1, "wor", 3); - fieldstat_hll_add(instance, 0, 0, &TEST_TAG_INT, 1, "world", 5); + fieldstat_hll_add(instance, 0, 0, &TEST_FIELD_INT, 1, "hello", 5); + fieldstat_hll_add(instance, 0, 0, &TEST_FIELD_INT, 1, "wor", 3); + fieldstat_hll_add(instance, 0, 0, &TEST_FIELD_INT, 1, "world", 5); struct fieldstat *instance_total = fieldstat_new(); fieldstat_merge(instance_total, instance); @@ -79,14 +113,14 @@ TEST(metric_test_hll, merge) TEST(metric_test_hll, merge_twice_with_reset) { struct fieldstat *instance = test_init_standard_instance_one_cube_one_metric_one_cell_hll(); - fieldstat_hll_add(instance, 0, 0, &TEST_TAG_INT, 1, "hello", 5); - fieldstat_hll_add(instance, 0, 0, &TEST_TAG_INT, 1, "wor", 3); + fieldstat_hll_add(instance, 0, 0, &TEST_FIELD_INT, 1, "hello", 5); + fieldstat_hll_add(instance, 0, 0, &TEST_FIELD_INT, 1, "wor", 3); struct fieldstat *instance_total = fieldstat_new(); fieldstat_merge(instance_total, instance); fieldstat_reset(instance); - fieldstat_hll_add(instance, 0, 0, &TEST_TAG_INT, 1, "world", 5); + fieldstat_hll_add(instance, 0, 0, &TEST_FIELD_INT, 1, "world", 5); fieldstat_merge(instance_total, instance); test_assert_standard_instance(instance_total); @@ -102,17 +136,17 @@ TEST(metric_test_hll, merge_twice_with_reset) (((uint32_t)(A) & 0x0000ff00) << 8) | \ (((uint32_t)(A) & 0x000000ff) << 24)) #include "base64/b64.h" -#include "st_hyperloglog.h" +#include "hyperloglog.h" TEST(metric_test_hll, serialize_with_b64_and_query) { struct fieldstat *instance = test_init_standard_instance_one_cube_one_metric_one_cell_hll(); - fieldstat_hll_add(instance, 0, 0, &TEST_TAG_INT, 1, "hello", 5); - fieldstat_hll_add(instance, 0, 0, &TEST_TAG_INT, 1, "wor", 3); - fieldstat_hll_add(instance, 0, 0, &TEST_TAG_INT, 1, "world", 5); + fieldstat_hll_add(instance, 0, 0, &TEST_FIELD_INT, 1, "hello", 5); + fieldstat_hll_add(instance, 0, 0, &TEST_FIELD_INT, 1, "wor", 3); + fieldstat_hll_add(instance, 0, 0, &TEST_FIELD_INT, 1, "world", 5); char *blob = NULL; size_t blob_len = 0; - fieldstat_get_serialized_blob(instance, 0, 0, &TEST_TAG_LIST_INT, &blob, &blob_len); + fieldstat_metric_get_serialization_as_base64(instance, 0, 0, &TEST_FIELD_LIST_INT, &blob, &blob_len); size_t dec_size = 0; unsigned char *dec = b64_decode_ex(blob, blob_len, &dec_size); @@ -123,7 +157,7 @@ TEST(metric_test_hll, serialize_with_b64_and_query) memcpy(&precision, dec + sizeof(unsigned char), sizeof(unsigned char)); EXPECT_EQ(precision, 10); // the one initialized in test_init_standard_instance_one_cube_one_metric_one_cell_hll - struct ST_hyperloglog *hll_from_blob = ST_hyperloglog_new(precision); + struct hyperloglog *hll_from_blob = hyperloglog_new(precision); int num_reg = NUM_REG(precision); int words = INT_CEIL(num_reg, REG_PER_WORD); size_t reg_size = words * sizeof(uint32_t); @@ -136,11 +170,11 @@ TEST(metric_test_hll, serialize_with_b64_and_query) memcpy(hll_from_blob->registers, registers, reg_size); free(registers); - EXPECT_NEAR(ST_hyperloglog_count(hll_from_blob), 3, 0.5); + EXPECT_NEAR(hyperloglog_count(hll_from_blob), 3, 0.5); free(blob); free(dec); fieldstat_free(instance); - ST_hyperloglog_free(hll_from_blob); + hyperloglog_free(hll_from_blob); } @@ -151,32 +185,32 @@ extern "C" { TEST(metric_test_hll, serialize_with_b64_and_query_with_python_api) { struct fieldstat *instance = test_init_standard_instance_one_cube_one_metric_one_cell_hll(); - fieldstat_hll_add(instance, 0, 0, &TEST_TAG_INT, 1, "hello", 5); - fieldstat_hll_add(instance, 0, 0, &TEST_TAG_INT, 1, "wor", 3); - fieldstat_hll_add(instance, 0, 0, &TEST_TAG_INT, 1, "world", 5); + fieldstat_hll_add(instance, 0, 0, &TEST_FIELD_INT, 1, "hello", 5); + fieldstat_hll_add(instance, 0, 0, &TEST_FIELD_INT, 1, "wor", 3); + fieldstat_hll_add(instance, 0, 0, &TEST_FIELD_INT, 1, "world", 5); char *blob = NULL; size_t blob_len = 0; - fieldstat_get_serialized_blob(instance, 0, 0, &TEST_TAG_LIST_INT, &blob, &blob_len); + fieldstat_metric_get_serialization_as_base64(instance, 0, 0, &TEST_FIELD_LIST_INT, &blob, &blob_len); bool flag = fieldstat_is_hll(blob); EXPECT_EQ(flag, true); void *hll_from_blob = hll_base64_decode(blob); - EXPECT_NEAR(ST_hyperloglog_count((struct ST_hyperloglog *)hll_from_blob), 3, 0.5); + EXPECT_NEAR(hyperloglog_count((struct hyperloglog *)hll_from_blob), 3, 0.5); free(blob); fieldstat_free(instance); - ST_hyperloglog_free((struct ST_hyperloglog *)hll_from_blob); + hyperloglog_free((struct hyperloglog *)hll_from_blob); } TEST(metric_test_hll, add_with_wrong_cube_id_expecting_fail) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_FIELD_INT_2, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int ret = fieldstat_hll_add(instance, cube_id + 1, 0, &TEST_TAG_INT, 1, "hello", 5); + int ret = fieldstat_hll_add(instance, cube_id + 1, 0, &TEST_FIELD_INT, 1, "hello", 5); EXPECT_EQ(ret, FS_ERR_INVALID_CUBE_ID); - ret = fieldstat_hll_add(instance, -1, 0, &TEST_TAG_INT, 1, "hello", 5); + ret = fieldstat_hll_add(instance, -1, 0, &TEST_FIELD_INT, 1, "hello", 5); EXPECT_EQ(ret, FS_ERR_INVALID_CUBE_ID); fieldstat_free(instance); @@ -185,20 +219,83 @@ TEST(metric_test_hll, add_with_wrong_cube_id_expecting_fail) TEST(metric_test_hll, add_with_wrong_metric_id_expecting_fail) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int metric_id = fieldstat_register_hll(instance, "czz_test hll metric", 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_FIELD_INT_2, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int metric_id = fieldstat_register_hll(instance, cube_id, "czz_test hll metric", 10); - int ret = fieldstat_hll_add(instance, cube_id, metric_id + 1, &TEST_TAG_INT, 1, "hello", 5); + int ret = fieldstat_hll_add(instance, cube_id, metric_id + 1, &TEST_FIELD_INT, 1, "hello", 5); EXPECT_EQ(ret, FS_ERR_INVALID_METRIC_ID); - ret = fieldstat_hll_add(instance, cube_id, -1, &TEST_TAG_INT, 1, "hello", 5); + ret = fieldstat_hll_add(instance, cube_id, -1, &TEST_FIELD_INT, 1, "hello", 5); EXPECT_EQ(ret, FS_ERR_INVALID_METRIC_ID); fieldstat_free(instance); } +TEST(metric_test_hll, spread_sketch_add_and_test_accuracy) +{ + struct fieldstat *instance = fieldstat_new(); + int K = 10; + fieldstat_cube_create(instance, &TEST_FIELD_INT_2, 1); + fieldstat_register_hll(instance, 0, "testss", 6); + fieldstat_cube_set_sampling(instance, 0, SAMPLING_MODE_TOP_CARDINALITY, K, 0); + + int n_flows = 100000; + std::unordered_map<std::string, std::unordered_set<std::string>> flow_cnt; + SpreadSketchZipfGenerator generator(1.0, K * 10); // give much bigger distribution, so that we can test the accuracy + for (int i = 0; i < n_flows; i++) + { + Flow f = generator.next(); + Fieldstat_tag_list_wrapper dimension("src ip", f.src_ip.c_str()); + Fieldstat_tag_list_wrapper counted("dst ip", f.dst_ip.c_str()); + + fieldstat_hll_add_field(instance, 0, 0, dimension.get_tag(), dimension.get_tag_count(), counted.get_tag(), counted.get_tag_count()); + + flow_cnt[dimension.to_string()].insert(counted.to_string()); + } + + // recall + std::unordered_map<std::string, int> expected_unique_cnt; + std::vector<struct Fieldstat_tag_list_wrapper *> test_result; + for (auto &kv : flow_cnt) { + expected_unique_cnt[kv.first] = kv.second.size(); + } + + struct field_list *tag_list = NULL; + size_t n_cell = 0; + fieldstat_cube_get_cells(instance, 0, &tag_list, &n_cell); + EXPECT_EQ(n_cell, K); + for (size_t i = 0; i < n_cell; i++) { + Fieldstat_tag_list_wrapper tmp = Fieldstat_tag_list_wrapper(&tag_list[i]); + test_result.push_back(new Fieldstat_tag_list_wrapper(tmp)); + } + double recall = test_cal_topk_accuracy(test_result, expected_unique_cnt); + printf("spread_sketch_add_and_test_accuracy recall: %f\n", recall); + EXPECT_GE(recall, 0.8); + + // MRE + double mre = 0; + for (size_t i = 0; i < n_cell; i++) { + Fieldstat_tag_list_wrapper tmp = Fieldstat_tag_list_wrapper(&tag_list[i]); + double value_true = expected_unique_cnt[tmp.to_string()]; + double value_est; + fieldstat_hll_get(instance, 0, &tag_list[i], 0, &value_est); + // printf("the estimated value for %s is %f, the true value is %f\n", tmp.to_string().c_str(), value_est, value_true); + + mre += fabs(value_true - value_est) / value_true; + } + mre = mre / n_cell; + printf("topk_add_and_test_accuracy Mean ratio e: %f\n", mre); + EXPECT_LE(mre, 0.2); + + fieldstat_field_list_arr_free(tag_list, n_cell); + fieldstat_free(instance); + for (auto &ptr : test_result) { + delete ptr; + } +} int main(int argc, char *argv[]) { testing::InitGoogleTest(&argc, argv); + // testing::GTEST_FLAG(filter) = "metric_test_hll.spread_sketch_add_and_test_accuracy"; return RUN_ALL_TESTS(); } diff --git a/test/test_performance.cpp b/test/test_performance.cpp index 11c05ce..48d3859 100644 --- a/test/test_performance.cpp +++ b/test/test_performance.cpp @@ -6,22 +6,32 @@ #include "fieldstat_exporter.h" #include "utils.hpp" + +int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *tag, size_t tag_count, enum sampling_mode mode, int k, int primary_metric_id=0) +{ + int ret = fieldstat_cube_create(instance, tag, tag_count); + fieldstat_cube_set_sampling(instance, ret, mode, k, primary_metric_id); + return ret; +} + + // /* -------------------------------------------------------------------------- */ // /* merge */ // /* -------------------------------------------------------------------------- */ -double perform_merge_test(std::function<void (struct fieldstat*, int, int, const struct fieldstat_tag *, int)> metric_add_func, +double perform_merge_test(std::function<void (struct fieldstat*, int, int, const struct field *, int)> metric_add_func, std::function<int(struct fieldstat*)> metric_register_func, enum sampling_mode mode, bool merge_empty_dest) { const int MAX_CELL_NUM = 1000; - Fieldstat_tag_list_wrapper *tags[MAX_CELL_NUM]; + Fieldstat_tag_list_wrapper *fields[MAX_CELL_NUM]; for (int i = 0; i < MAX_CELL_NUM; i++) { - tags[i] = new Fieldstat_tag_list_wrapper("my key", i); + fields[i] = new Fieldstat_tag_list_wrapper("my key", i); } struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, mode, MAX_CELL_NUM); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); int metric_id = metric_register_func(instance); + fieldstat_cube_set_sampling(instance, cube_id, mode, MAX_CELL_NUM, metric_id); for (int j = 0; j < MAX_CELL_NUM; j++) { - metric_add_func(instance, cube_id, metric_id, tags[j]->get_tag(), 1); + metric_add_func(instance, cube_id, metric_id, fields[j]->get_tag(), 1); } struct fieldstat *instance_dest = fieldstat_new(); @@ -36,7 +46,7 @@ double perform_merge_test(std::function<void (struct fieldstat*, int, int, const fieldstat_free(instance_dest); fieldstat_free(instance); for (int i = 0; i < MAX_CELL_NUM; i++) { - delete tags[i]; + delete fields[i]; } return (end - start) / 1000.0; @@ -44,11 +54,11 @@ double perform_merge_test(std::function<void (struct fieldstat*, int, int, const TEST(test_performance, merge_performance_one_instance_comprehensive_counter_empty_dest) { - auto metric_add_func = [](struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag *tags, int n_tag) { - fieldstat_counter_incrby(instance, cube_id, metric_id, tags, n_tag, 1); + auto metric_add_func = [](struct fieldstat *instance, int cube_id, int metric_id, const struct field *fields, int n_field) { + fieldstat_counter_incrby(instance, cube_id, metric_id, fields, n_field, 1); }; auto metric_register_func = [](struct fieldstat *instance) { - return fieldstat_register_counter(instance, "metric name"); + return fieldstat_register_counter(instance, 0, "metric name"); }; double elapsed = perform_merge_test(metric_add_func, metric_register_func, SAMPLING_MODE_COMPREHENSIVE, true); @@ -61,11 +71,11 @@ TEST(test_performance, merge_performance_one_instance_comprehensive_hll_empty_de // int metric_id = fieldstat_register_hll(instance, cube_id, "czz_test hll metric", 10); // int ret = fieldstat_hll_add(instance, cube_id, metric_id, cell_id, "hello", 5); - auto metric_add_func = [](struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag *tags, int n_tag) { - fieldstat_hll_add(instance, cube_id, metric_id, tags, n_tag, "hello", 5); + auto metric_add_func = [](struct fieldstat *instance, int cube_id, int metric_id, const struct field *fields, int n_field) { + fieldstat_hll_add(instance, cube_id, metric_id, fields, n_field, "hello", 5); }; auto metric_register_func = [](struct fieldstat *instance) { - return fieldstat_register_hll(instance, "hll metric", 6); + return fieldstat_register_hll(instance, 0, "hll metric", 6); }; double elapsed = perform_merge_test(metric_add_func, metric_register_func, SAMPLING_MODE_COMPREHENSIVE, true); @@ -75,14 +85,11 @@ TEST(test_performance, merge_performance_one_instance_comprehensive_hll_empty_de TEST(test_performance, merge_performance_one_instance_comprehensive_histogram_empty_dest) { - // int metric_id = fieldstat_register_hist(instance, cube_id, "czz_test", 1, 100000, 1); - // int ret = fieldstat_hist_record(instance, cube_id, metric_id, cell_id, 1234); - - auto metric_add_func = [](struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag *tags, int n_tag) { - fieldstat_hist_record(instance, cube_id, metric_id, tags, n_tag, 1234); + auto metric_add_func = [](struct fieldstat *instance, int cube_id, int metric_id, const struct field *fields, int n_field) { + fieldstat_histogram_record(instance, cube_id, metric_id, fields, n_field, 1234); }; auto metric_register_func = [](struct fieldstat *instance) { - return fieldstat_register_hist(instance, "histogram metric", 1, 100000, 1); + return fieldstat_register_histogram(instance, 0, "histogram metric", 1, 100000, 1); }; double elapsed = perform_merge_test(metric_add_func, metric_register_func, SAMPLING_MODE_COMPREHENSIVE, true); @@ -92,11 +99,11 @@ TEST(test_performance, merge_performance_one_instance_comprehensive_histogram_em TEST(test_performance, merge_performance_one_instance_topk_counter_empty_dest) { - auto metric_add_func = [](struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag *tags, int n_tag) { - fieldstat_counter_incrby(instance, cube_id, metric_id, tags, n_tag, rand() % 1000); + auto metric_add_func = [](struct fieldstat *instance, int cube_id, int metric_id, const struct field *fields, int n_field) { + fieldstat_counter_incrby(instance, cube_id, metric_id, fields, n_field, rand() % 1000); }; auto metric_register_func = [](struct fieldstat *instance) { - return fieldstat_register_counter(instance, "metric name"); + return fieldstat_register_counter(instance, 0, "metric name"); }; double elapsed = perform_merge_test(metric_add_func, metric_register_func, SAMPLING_MODE_TOPK, true); @@ -106,11 +113,11 @@ TEST(test_performance, merge_performance_one_instance_topk_counter_empty_dest) TEST(test_performance, merge_performance_one_instance_comprehensive_counter_full_dest) { - auto metric_add_func = [](struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag *tags, int n_tag) { - fieldstat_counter_incrby(instance, cube_id, metric_id, tags, n_tag, 1); + auto metric_add_func = [](struct fieldstat *instance, int cube_id, int metric_id, const struct field *fields, int n_field) { + fieldstat_counter_incrby(instance, cube_id, metric_id, fields, n_field, 1); }; auto metric_register_func = [](struct fieldstat *instance) { - return fieldstat_register_counter(instance, "metric name"); + return fieldstat_register_counter(instance, 0, "metric name"); }; double elapsed = perform_merge_test(metric_add_func, metric_register_func, SAMPLING_MODE_COMPREHENSIVE, false); @@ -120,11 +127,11 @@ TEST(test_performance, merge_performance_one_instance_comprehensive_counter_full TEST(test_performance, merge_performance_one_instance_comprehensive_hll_full_dest) { - auto metric_add_func = [](struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag *tags, int n_tag) { - fieldstat_hll_add(instance, cube_id, metric_id, tags, n_tag, "hello", 5); + auto metric_add_func = [](struct fieldstat *instance, int cube_id, int metric_id, const struct field *fields, int n_field) { + fieldstat_hll_add(instance, cube_id, metric_id, fields, n_field, "hello", 5); }; auto metric_register_func = [](struct fieldstat *instance) { - return fieldstat_register_hll(instance, "hll metric", 6); + return fieldstat_register_hll(instance, 0, "hll metric", 6); }; double elapsed = perform_merge_test(metric_add_func, metric_register_func, SAMPLING_MODE_COMPREHENSIVE, false); @@ -134,11 +141,11 @@ TEST(test_performance, merge_performance_one_instance_comprehensive_hll_full_des TEST(test_performance, merge_performance_one_instance_comprehensive_histogram_full_dest) { - auto metric_add_func = [](struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag *tags, int n_tag) { - fieldstat_hist_record(instance, cube_id, metric_id, tags, n_tag, 1234); + auto metric_add_func = [](struct fieldstat *instance, int cube_id, int metric_id, const struct field *fields, int n_field) { + fieldstat_histogram_record(instance, cube_id, metric_id, fields, n_field, 1234); }; auto metric_register_func = [](struct fieldstat *instance) { - return fieldstat_register_hist(instance, "histogram metric", 1, 100000, 1); + return fieldstat_register_histogram(instance, 0, "histogram metric", 1, 100000, 1); }; double elapsed = perform_merge_test(metric_add_func, metric_register_func, SAMPLING_MODE_COMPREHENSIVE, false); @@ -148,11 +155,11 @@ TEST(test_performance, merge_performance_one_instance_comprehensive_histogram_fu TEST(test_performance, merge_performance_one_instance_topk_counter_full_dest) { - auto metric_add_func = [](struct fieldstat *instance, int cube_id, int metric_id, const struct fieldstat_tag *tags, int n_tag) { - fieldstat_counter_incrby(instance, cube_id, metric_id, tags, n_tag, rand() % 1000); + auto metric_add_func = [](struct fieldstat *instance, int cube_id, int metric_id, const struct field *fields, int n_field) { + fieldstat_counter_incrby(instance, cube_id, metric_id, fields, n_field, rand() % 1000); }; auto metric_register_func = [](struct fieldstat *instance) { - return fieldstat_register_counter(instance, "metric name"); + return fieldstat_register_counter(instance,0, "metric name"); }; double elapsed = perform_merge_test(metric_add_func, metric_register_func, SAMPLING_MODE_TOPK, false); @@ -163,14 +170,16 @@ TEST(test_performance, merge_performance_one_instance_topk_counter_full_dest) struct fieldstat *construct_fs_with_many_empty_cubes(int cube_num, int metric_num, enum sampling_mode mode) { struct fieldstat *instance = fieldstat_new(); - struct fieldstat_tag tmp_tag = TEST_TAG_INT; + struct field tmp_tag = TEST_FIELD_INT; for (int i = 0; i < cube_num; i++) { tmp_tag.value_longlong = i; - fieldstat_create_cube(instance, &tmp_tag, 1, mode, 1000); + int cube_id = fieldstat_cube_create(instance, &tmp_tag, 1); for (int j = 0; j < metric_num; j++) { - fieldstat_register_counter(instance, std::to_string(j).c_str()); + fieldstat_register_counter(instance, cube_id, std::to_string(j).c_str()); } + + fieldstat_cube_set_sampling(instance, cube_id, mode, 1000, 0); } return instance; } @@ -215,19 +224,19 @@ TEST(test_performance, merge_empty_cubes_topk) TEST(test_performance, performance_test_add_cells_comprehensive) { size_t cell_count = 100000; - struct fieldstat_tag tags[cell_count]; + struct field fields[cell_count]; for (size_t i = 0; i < cell_count; i++) { - tags[i] = TEST_TAG_INT; - tags[i].value_longlong = i; + fields[i] = TEST_FIELD_INT; + fields[i].value_longlong = i; } // getchar(); struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, cell_count); - fieldstat_register_counter(instance, "test"); + test_fieldstat_cube_create(instance, &TEST_FIELD_INT_2, 1, SAMPLING_MODE_COMPREHENSIVE, cell_count); + fieldstat_register_counter(instance, 0, "test"); clock_t start = clock(); for (size_t i = 0; i < cell_count; i++) { - fieldstat_counter_incrby(instance, 0, 0, &tags[i % cell_count], 1, 1); + fieldstat_counter_incrby(instance, 0, 0, &fields[i % cell_count], 1, 1); } clock_t end = clock(); double seconds = (double)(end - start) / cell_count; @@ -239,23 +248,24 @@ TEST(test_performance, performance_test_add_cells_comprehensive) TEST(test_performance, performance_test_add_cells_topk) { size_t cell_count = 100000; - struct fieldstat_tag tags[cell_count]; + struct field fields[cell_count]; for (size_t i = 0; i < cell_count; i++) { - tags[i] = TEST_TAG_INT; - // tags[i].value_longlong = rand() % 10000; + fields[i] = TEST_FIELD_INT; + // fields[i].value_longlong = rand() % 10000; if (rand()%2) - tags[i].value_longlong = i; + fields[i].value_longlong = i; else - tags[i].value_longlong = rand() % 1000; + fields[i].value_longlong = rand() % 1000; } struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_TOPK, 1000); - fieldstat_register_counter(instance, "test"); + fieldstat_cube_create(instance, &TEST_FIELD_INT_2, 1); + fieldstat_register_counter(instance, 0, "test"); + fieldstat_cube_set_sampling(instance, 0, SAMPLING_MODE_TOPK, 1000, 0); // getchar(); clock_t start = clock(); for (size_t i = 0; i < cell_count; i++) { - fieldstat_counter_incrby(instance, 0, 0, &tags[i % cell_count], 1, 1); + fieldstat_counter_incrby(instance, 0, 0, &fields[i % cell_count], 1, 1); } clock_t end = clock(); double seconds = (double)(end - start) / cell_count; @@ -270,8 +280,8 @@ TEST(test_performance, performance_test_add_cells_topk) TEST(test_performance, performance_test_add_cells_histogram_record) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - fieldstat_register_hist(instance, "test", 1, 100000, 3); + test_fieldstat_cube_create(instance, &TEST_FIELD_INT_2, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + fieldstat_register_histogram(instance, 0, "test", 1, 100000, 3); size_t test_num = 100000; long long vals[test_num]; for (size_t i = 0; i < test_num; i++) { @@ -279,7 +289,7 @@ TEST(test_performance, performance_test_add_cells_histogram_record) } clock_t start = clock(); for (size_t i = 0; i < test_num; i++) { - fieldstat_hist_record(instance, 0, 0, &TEST_TAG_INT, 1, vals[i]); + fieldstat_histogram_record(instance, 0, 0, &TEST_FIELD_INT, 1, vals[i]); } clock_t end = clock(); double seconds = (double)(end - start) / test_num; @@ -291,8 +301,8 @@ TEST(test_performance, performance_test_add_cells_histogram_record) TEST(test_performance, performance_test_add_cells_hll_add) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - fieldstat_register_hll(instance, "test", 6); + test_fieldstat_cube_create(instance, &TEST_FIELD_INT_2, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + fieldstat_register_hll(instance, 0, "test", 6); size_t test_num = 100000; std::string vals[test_num]; for (size_t i = 0; i < test_num; i++) { @@ -301,7 +311,7 @@ TEST(test_performance, performance_test_add_cells_hll_add) clock_t start = clock(); for (size_t i = 0; i < test_num; i++) { - fieldstat_hll_add(instance, 0, 0, &TEST_TAG_INT, 1, vals[i].c_str(), vals[i].length()); + fieldstat_hll_add(instance, 0, 0, &TEST_FIELD_INT, 1, vals[i].c_str(), vals[i].length()); } clock_t end = clock(); double seconds = (double)(end - start) / test_num; @@ -313,23 +323,23 @@ TEST(test_performance, performance_test_add_cells_hll_add) TEST(test_performance, performance_test_add_cells_comprehensive_5_tags) { size_t cell_count = 100000; - struct fieldstat_tag *tag_v[cell_count]; + struct field *tag_v[cell_count]; for (size_t i = 0; i < cell_count; i++) { - struct fieldstat_tag *tags = (struct fieldstat_tag *)malloc(sizeof(struct fieldstat_tag) * 5); + struct field *fields = (struct field *)malloc(sizeof(struct field) * 5); - tags[0] = TEST_TAG_INT; - tags[1] = TEST_TAG_INT; - tags[2] = TEST_TAG_INT; - tags[3] = TEST_TAG_INT; - tags[4] = TEST_TAG_INT; - tags[0].value_longlong = i; + fields[0] = TEST_FIELD_INT; + fields[1] = TEST_FIELD_INT; + fields[2] = TEST_FIELD_INT; + fields[3] = TEST_FIELD_INT; + fields[4] = TEST_FIELD_INT; + fields[0].value_longlong = i; - tag_v[i] = tags; + tag_v[i] = fields; } // getchar(); struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, cell_count); - fieldstat_register_counter(instance, "test"); + test_fieldstat_cube_create(instance, &TEST_FIELD_INT_2, 1, SAMPLING_MODE_COMPREHENSIVE, cell_count); + fieldstat_register_counter(instance, 0, "test"); clock_t start = clock(); for (size_t i = 0; i < cell_count; i++) { @@ -337,7 +347,7 @@ TEST(test_performance, performance_test_add_cells_comprehensive_5_tags) } clock_t end = clock(); double seconds = (double)(end - start) / cell_count; - printf("performance_test_add_cells_comprehensive time 5 tags cost: %f\n", seconds); + printf("performance_test_add_cells_comprehensive time 5 fields cost: %f\n", seconds); EXPECT_TRUE(seconds < 2); fieldstat_free(instance); @@ -349,27 +359,27 @@ TEST(test_performance, performance_test_add_cells_comprehensive_5_tags) TEST(test_performance, performance_test_add_cells_histogram_record_5tags) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - fieldstat_register_hist(instance, "test", 1, 100000, 3); + test_fieldstat_cube_create(instance, &TEST_FIELD_INT_2, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + fieldstat_register_histogram(instance, 0, "test", 1, 100000, 3); size_t test_num = 100000; long long vals[test_num]; for (size_t i = 0; i < test_num; i++) { vals[i] = rand() % 100000 + 1; } - struct fieldstat_tag tags[5]; - tags[0] = TEST_TAG_INT; - tags[1] = TEST_TAG_STRING; - tags[2] = TEST_TAG_DOUBLE; - tags[3] = TEST_TAG_INT; - tags[4] = TEST_TAG_INT; + struct field fields[5]; + fields[0] = TEST_FIELD_INT; + fields[1] = TEST_FIELD_STRING; + fields[2] = TEST_FIELD_DOUBLE; + fields[3] = TEST_FIELD_INT; + fields[4] = TEST_FIELD_INT; clock_t start = clock(); for (size_t i = 0; i < test_num; i++) { - fieldstat_hist_record(instance, 0, 0, tags, 5, vals[i]); + fieldstat_histogram_record(instance, 0, 0, fields, 5, vals[i]); } clock_t end = clock(); double seconds = (double)(end - start) / test_num; - printf("performance_test_add_cells_histogram_record time 5 tags cost: %f\n", seconds); + printf("performance_test_add_cells_histogram_record time 5 fields cost: %f\n", seconds); EXPECT_TRUE(seconds < 1); fieldstat_free(instance); } @@ -377,27 +387,27 @@ TEST(test_performance, performance_test_add_cells_histogram_record_5tags) TEST(test_performance, performance_test_add_cells_hll_add_5tags) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - fieldstat_register_hll(instance, "test", 6); + test_fieldstat_cube_create(instance, &TEST_FIELD_INT_2, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + fieldstat_register_hll(instance, 0, "test", 6); size_t test_num = 100000; std::string vals[test_num]; for (size_t i = 0; i < test_num; i++) { vals[i] = std::to_string(rand() % 1000000 + 1); } - struct fieldstat_tag tags[5]; - tags[0] = TEST_TAG_INT; - tags[1] = TEST_TAG_STRING; - tags[2] = TEST_TAG_DOUBLE; - tags[3] = TEST_TAG_INT; - tags[4] = TEST_TAG_INT; + struct field fields[5]; + fields[0] = TEST_FIELD_INT; + fields[1] = TEST_FIELD_STRING; + fields[2] = TEST_FIELD_DOUBLE; + fields[3] = TEST_FIELD_INT; + fields[4] = TEST_FIELD_INT; clock_t start = clock(); for (size_t i = 0; i < test_num; i++) { - fieldstat_hll_add(instance, 0, 0, tags, 5, vals[i].c_str(), vals[i].length()); + fieldstat_hll_add(instance, 0, 0, fields, 5, vals[i].c_str(), vals[i].length()); } clock_t end = clock(); double seconds = (double)(end - start) / test_num; - printf("performance_test_add_cells_hll_add time 5 tags cost: %f\n", seconds); + printf("performance_test_add_cells_hll_add time 5 fields cost: %f\n", seconds); EXPECT_TRUE(seconds < 1); fieldstat_free(instance); } @@ -414,22 +424,22 @@ TEST(test_performance, export_many_cells) const int CUBE_NUM = 10; const int METRIC_NUM = 10; - Fieldstat_tag_list_wrapper *tags[TAG_NUM]; + Fieldstat_tag_list_wrapper *fields[TAG_NUM]; for (int i = 0; i < TAG_NUM; i++) { - tags[i] = new Fieldstat_tag_list_wrapper("my key", i); + fields[i] = new Fieldstat_tag_list_wrapper("my key", i); } struct fieldstat *instance = fieldstat_new(); for (int i = 0; i < CUBE_NUM; i++) { Fieldstat_tag_list_wrapper cube_tag("shared key", i); - int cube_id = fieldstat_create_cube(instance, cube_tag.get_tag(), cube_tag.get_tag_count(), SAMPLING_MODE_COMPREHENSIVE, MAX_CELL_NUM); + int cube_id = test_fieldstat_cube_create(instance, cube_tag.get_tag(), cube_tag.get_tag_count(), SAMPLING_MODE_COMPREHENSIVE, MAX_CELL_NUM); for (int j = 0; j < METRIC_NUM; j++) { string metric_name = "metric name" + to_string(i) + to_string(j); - int metric_id = fieldstat_register_counter(instance, metric_name.c_str()); + int metric_id = fieldstat_register_counter(instance, cube_id, metric_name.c_str()); for (int k = 0; k < MAX_CELL_NUM; k++) { - fieldstat_counter_incrby(instance, cube_id, metric_id, tags[rand() % TAG_NUM]->get_tag(), 1, 1); + fieldstat_counter_incrby(instance, cube_id, metric_id, fields[rand() % TAG_NUM]->get_tag(), 1, 1); } } } @@ -447,7 +457,7 @@ TEST(test_performance, export_many_cells) fieldstat_free(instance); for (int i = 0; i < TAG_NUM; i++) { - delete tags[i]; + delete fields[i]; } } @@ -496,12 +506,12 @@ TEST(test_performance, callibrate_unchanged) fieldstat_free(instance_dest); } -struct fieldstat *construct_fs_with_many_cells(int cell_num, enum sampling_mode mode) +struct fieldstat *construct_fs_with_many_cells(int cell_num) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_TAG_INT, 1, mode, cell_num); - fieldstat_register_counter(instance, "test"); - struct fieldstat_tag tmp_tag = TEST_TAG_INT; + test_fieldstat_cube_create(instance, &TEST_FIELD_INT, 1, SAMPLING_MODE_COMPREHENSIVE, cell_num); + fieldstat_register_counter(instance, 0, "test"); + struct field tmp_tag = TEST_FIELD_INT; for (int i = 0; i < cell_num; i++) { tmp_tag.value_longlong = i; fieldstat_counter_incrby(instance, 0, 0, &tmp_tag, 1, 1); @@ -512,7 +522,7 @@ struct fieldstat *construct_fs_with_many_cells(int cell_num, enum sampling_mode TEST(test_performance, reset_many_cells) { - struct fieldstat *instance = construct_fs_with_many_cells(1000, SAMPLING_MODE_COMPREHENSIVE); // many empty cubes, 10 metrics is a common case + struct fieldstat *instance = construct_fs_with_many_cells(1000); // many empty cubes, 10 metrics is a common case clock_t start = clock(); fieldstat_reset(instance); clock_t end = clock(); diff --git a/test/test_register_and_reset.cpp b/test/test_register_and_reset.cpp index 57c947e..b79da54 100644 --- a/test/test_register_and_reset.cpp +++ b/test/test_register_and_reset.cpp @@ -3,137 +3,345 @@ #include "fieldstat.h" #include "utils.hpp" +int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k) +{ + assert(mode == SAMPLING_MODE_COMPREHENSIVE); + int ret = fieldstat_cube_create(instance, dimensions, n_dimensions); + fieldstat_cube_set_sampling(instance, ret, mode, k, 0); + return ret; +} -TEST(test_register, reset_and_version) +TEST(test_register, delete_cube_and_register_and_origin_position) { struct fieldstat *instance = fieldstat_new(); - EXPECT_EQ(fieldstat_get_version(instance), 0); - fieldstat_reset(instance); - EXPECT_EQ(fieldstat_get_version(instance), 1); - fieldstat_reset(instance); - fieldstat_reset(instance); - EXPECT_EQ(fieldstat_get_version(instance), 3); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + fieldstat_cube_destroy(instance, cube_id); + cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + EXPECT_EQ(cube_id, 0); + EXPECT_EQ(fieldstat_find_cube(instance, &TEST_SHARED_TAG, 1) , 0); + fieldstat_free(instance); +} + +TEST(test_register, delete_comprehensive_cube_with_cells_and_metrics) +{ + struct fieldstat *instance = fieldstat_new(); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, 1); + + fieldstat_cube_destroy(instance, cube_id); + + struct field_list *tag_list = fieldstat_cube_get_dimensions(instance, cube_id); + EXPECT_EQ(tag_list, nullptr); + int cube_id_ret = fieldstat_find_cube(instance, &TEST_SHARED_TAG, 1); + EXPECT_EQ(cube_id_ret, FS_ERR_INVALID_DIMENSION); fieldstat_free(instance); } -TEST(test_register, delete_cube_and_version_increase) +TEST(test_register, delete_topk_cube_with_cells_and_metrics) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - EXPECT_EQ(fieldstat_get_cube_version(instance, cube_id), 0); - int ret = fieldstat_destroy_cube(instance, cube_id); - EXPECT_EQ(ret, 0); - EXPECT_EQ(fieldstat_get_cube_version(instance, cube_id), FS_ERR_INVALID_CUBE_ID); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, metric_id); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, 1); - cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - EXPECT_EQ(fieldstat_get_cube_version(instance, cube_id), 1); + fieldstat_cube_destroy(instance, cube_id); + struct field_list *tag_list = fieldstat_cube_get_dimensions(instance, cube_id); + EXPECT_EQ(tag_list, nullptr); + int cube_id_ret = fieldstat_find_cube(instance, &TEST_SHARED_TAG, 1); + EXPECT_EQ(cube_id_ret, FS_ERR_INVALID_DIMENSION); fieldstat_free(instance); } -TEST(test_register, query_on_wrong_version) +TEST(test_register, delete_spreadsketch_cube_with_cells_and_metrics) { - EXPECT_EQ(fieldstat_get_cube_version(NULL, 1), FS_ERR_NULL_HANDLER); struct fieldstat *instance = fieldstat_new(); - EXPECT_EQ(fieldstat_get_cube_version(instance, 1), FS_ERR_INVALID_CUBE_ID); - EXPECT_EQ(fieldstat_get_cube_version(instance, -1), FS_ERR_INVALID_CUBE_ID); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id1 = fieldstat_register_counter(instance, cube_id, "counter"); + int metric_primary = fieldstat_register_hll(instance, cube_id, "hll_primary", 5); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 10, metric_primary); + + fieldstat_counter_incrby(instance, cube_id, metric_id1, &TEST_FIELD_INT, 1, 1); + fieldstat_hll_add_field(instance, cube_id, metric_primary, &TEST_FIELD_INT, 1, &TEST_FIELD_DOUBLE, 1); + + fieldstat_cube_destroy(instance, cube_id); + struct field_list *tag_list = fieldstat_cube_get_dimensions(instance, cube_id); + EXPECT_EQ(tag_list, nullptr); + int cube_id_ret = fieldstat_find_cube(instance, &TEST_SHARED_TAG, 1); + EXPECT_EQ(cube_id_ret, FS_ERR_INVALID_DIMENSION); + fieldstat_free(instance); } -TEST(test_register, delete_cube_and_register_and_origin_position) +int test_get_max_metric_id(const struct fieldstat *instance) +{ + int *metric_id_out; + size_t n_metric; + (void)fieldstat_cube_get_metrics(instance, 0, &metric_id_out, &n_metric); + free(metric_id_out); + return n_metric - 1; +} + +TEST(test_register, reset_and_try_to_query_cell_comprehensive) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - fieldstat_destroy_cube(instance, cube_id); - cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - EXPECT_EQ(cube_id, 0); - EXPECT_EQ(fieldstat_find_cube(instance, &TEST_SHARED_TAG, 1) , 0); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, 1); + + fieldstat_reset(instance); + long long value; + EXPECT_EQ(fieldstat_counter_get(instance, cube_id, &TEST_FIELD_LIST_INT, metric_id, &value), FS_ERR_INVALID_DIMENSION); + + size_t n_cell; + struct field_list *tag_list; + fieldstat_cube_get_cells(instance, cube_id, &tag_list, &n_cell); + EXPECT_EQ(n_cell, 0); + fieldstat_free(instance); } -TEST(test_register, delete_comprehensive_cube_with_cells_and_metrics) +TEST(test_register, reset_and_try_to_query_metric) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int metric_id = fieldstat_register_counter(instance, "counter"); - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int metric_counter = fieldstat_register_counter(instance, cube_id, "counter"); + int metric_hll = fieldstat_register_hll(instance, cube_id, "hll", 5); + int metric_hist = fieldstat_register_histogram(instance, cube_id, "hist", 0, 10000, 5); + int metric_operated_after_reset = fieldstat_register_counter(instance, cube_id, "counter_after_reset"); + + fieldstat_counter_incrby(instance, cube_id, metric_counter, &TEST_FIELD_INT, 1, 1); + fieldstat_hll_add(instance, cube_id, metric_counter, &TEST_FIELD_INT, 1, "1", 1); + fieldstat_histogram_record(instance, cube_id, metric_hist, &TEST_FIELD_INT, 1, 1000); + + fieldstat_reset(instance); + fieldstat_counter_incrby(instance, cube_id, metric_operated_after_reset, &TEST_FIELD_INT, 1, 1); + + long long value; + double value_hll; + EXPECT_EQ(fieldstat_counter_get(instance, cube_id, &TEST_FIELD_LIST_INT, metric_counter, &value), FS_ERR_INVALID_METRIC_ID); + EXPECT_EQ(fieldstat_hll_get(instance, cube_id, &TEST_FIELD_LIST_INT, metric_hll, &value_hll), FS_ERR_INVALID_METRIC_ID); + EXPECT_EQ(fieldstat_histogram_value_at_percentile(instance, cube_id, &TEST_FIELD_LIST_INT, metric_hist, 0.5), FS_ERR_INVALID_METRIC_ID); + EXPECT_EQ(fieldstat_counter_get(instance, cube_id, &TEST_FIELD_LIST_INT, metric_operated_after_reset, &value), FS_OK); - fieldstat_destroy_cube(instance, cube_id); fieldstat_free(instance); } -TEST(test_register, delete_topk_cube_with_cells_and_metrics) +TEST(test_register, reset_and_try_to_query_cell_topk) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); - int metric_id = fieldstat_register_counter(instance, "counter"); - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, metric_id); + + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, 1); + + fieldstat_reset(instance); + long long value; + EXPECT_EQ(fieldstat_counter_get(instance, cube_id, &TEST_FIELD_LIST_INT, metric_id, &value), FS_ERR_INVALID_DIMENSION); + + size_t n_cell; + struct field_list *tag_list; + fieldstat_cube_get_cells(instance, cube_id, &tag_list, &n_cell); + EXPECT_EQ(n_cell, 0); - fieldstat_destroy_cube(instance, cube_id); fieldstat_free(instance); } -int test_get_max_metric_id(const struct fieldstat *instance) +TEST(test_register, reset_and_try_to_query_cell_spreadsketch) { - int *metric_id_out; - size_t n_metric; - (void)fieldstat_get_metrics(instance, &metric_id_out, &n_metric); - free(metric_id_out); - return n_metric - 1; + struct fieldstat *instance = fieldstat_new(); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_hll(instance, cube_id, "hll", 5); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 10, metric_id); + fieldstat_hll_add(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, "12abc", 5); + + fieldstat_reset(instance); + double value; + EXPECT_EQ(fieldstat_hll_get(instance, cube_id, &TEST_FIELD_LIST_INT, metric_id, &value), FS_ERR_INVALID_DIMENSION); + + size_t n_cell; + struct field_list *tag_list; + fieldstat_cube_get_cells(instance, cube_id, &tag_list, &n_cell); + EXPECT_EQ(n_cell, 0); + + fieldstat_free(instance); } -TEST(test_register, reset_and_try_to_query_cell) +TEST(test_register, reset_and_new_cell_comprehensive) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int metric_id = fieldstat_register_counter(instance, "counter"); - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2); + int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, 1); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_DOUBLE, 1, 1); + int ret = fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_STRING, 1, 1); + EXPECT_EQ(ret, FS_ERR_TOO_MANY_CELLS); fieldstat_reset(instance); - EXPECT_EQ(test_get_max_metric_id(instance), 0); - long long value; - EXPECT_EQ(fieldstat_counter_get(instance, cube_id, metric_id, &TEST_TAG_LIST_INT, &value), FS_ERR_INVALID_TAG); + ret = fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_STRING, 1, 1); + EXPECT_EQ(ret, FS_OK); + + fieldstat_free(instance); +} + +TEST(test_register, reset_and_new_cell_topk) +{ + struct fieldstat *instance = fieldstat_new(); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 1, metric_id); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, 100);//100: bigger value + int ret = fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_DOUBLE, 1, 1); + EXPECT_EQ(ret, FS_ERR_TOO_MANY_CELLS); + + fieldstat_reset(instance); + ret = fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_DOUBLE, 1, 1); + EXPECT_EQ(ret, FS_OK); fieldstat_free(instance); } -TEST(test_register, reset_and_new_cell) +TEST(test_register, reset_and_new_cell_spreadsketch) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 2); - int metric_id = fieldstat_register_counter(instance, "counter"); - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1); - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_DOUBLE, 1, 1); - int ret = fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, 1); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_hll(instance, cube_id, "hll", 5); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, 1, metric_id); + // spread sketch will store more data than expected cell number 1. So loop for many cells first to trigger the error + struct field test_tag_long = TEST_FIELD_INT; + for (int i = 0; i < 10000; i++) { + test_tag_long.value_longlong = i; + fieldstat_hll_add(instance, cube_id, metric_id, &test_tag_long, 1, "12abc", 5); + } + int ret = fieldstat_hll_add(instance, cube_id, metric_id, &TEST_FIELD_DOUBLE, 1, "12abc", 5); EXPECT_EQ(ret, FS_ERR_TOO_MANY_CELLS); fieldstat_reset(instance); - ret = fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_STRING, 1, 1); + ret = fieldstat_hll_add(instance, cube_id, metric_id, &TEST_FIELD_DOUBLE, 1, "12abc", 5); EXPECT_EQ(ret, FS_OK); fieldstat_free(instance); } +TEST(test_register, ensure_recovery_more_faster_comprehensive) { + struct fieldstat *instance = fieldstat_new(); + int cell_num = 1000; + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, cell_num); + int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + struct field test_tag_long = TEST_FIELD_INT; + + clock_t start = clock(); + for (int i = 0; i < cell_num; i++) { + test_tag_long.value_longlong = i; + fieldstat_counter_incrby(instance, cube_id, metric_id, &test_tag_long, 1, 1); + } + clock_t end = clock(); + clock_t duration_initialize = end - start; + + fieldstat_reset(instance); + + start = clock(); + for (int i = 0; i < cell_num; i++) { + test_tag_long.value_longlong = i; + fieldstat_counter_incrby(instance, cube_id, metric_id, &test_tag_long, 1, 1); + } + end = clock(); + clock_t duration_reset = end - start; + + EXPECT_LT(duration_reset, duration_initialize); + + fieldstat_free(instance); +} + +TEST(test_register, ensure_recovery_more_faster_topk) { + struct fieldstat *instance = fieldstat_new(); + int cell_num = 1000; + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, cell_num, metric_id); + + struct field test_tag_long = TEST_FIELD_INT; + + clock_t start = clock(); + for (int i = 0; i < cell_num; i++) { + test_tag_long.value_longlong = i; + fieldstat_counter_incrby(instance, cube_id, metric_id, &test_tag_long, 1, 1); + } + clock_t end = clock(); + clock_t duration_initialize = end - start; + + fieldstat_reset(instance); + + start = clock(); + for (int i = 0; i < cell_num; i++) { + test_tag_long.value_longlong = i; + fieldstat_counter_incrby(instance, cube_id, metric_id, &test_tag_long, 1, 1); + } + end = clock(); + clock_t duration_reset = end - start; + + EXPECT_LT(duration_reset, duration_initialize); + + fieldstat_free(instance); +} + +TEST(test_register, ensure_recovery_more_faster_spreadsketch) { + struct fieldstat *instance = fieldstat_new(); + int cell_num = 1000; + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_hll(instance, cube_id, "counter", 6); + fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOP_CARDINALITY, cell_num, metric_id); + char long_string[1000]; + memset(long_string, 'a', 1000); + long_string[999] = '\0'; + struct field test_tag_long = {long_string, FIELD_VALUE_INTEGER, {.value_longlong = 0}}; + + clock_t start = clock(); + for (int i = 0; i < cell_num; i++) { + test_tag_long.value_longlong = i; + fieldstat_hll_add(instance, cube_id, metric_id, &test_tag_long, 1, "1", 1); + } + clock_t end = clock(); + clock_t duration_initialize = end - start; + + fieldstat_reset(instance); + + start = clock(); + for (int i = 0; i < cell_num; i++) { + test_tag_long.value_longlong = i; + fieldstat_hll_add(instance, cube_id, metric_id, &test_tag_long, 1, "1", 1); + } + end = clock(); + clock_t duration_reset = end - start; + + EXPECT_LT(duration_reset, duration_initialize); + + fieldstat_free(instance); +} + TEST(test_register, register_many_cubes) { struct fieldstat *instance = fieldstat_new(); int registered_cube = 10000; // will trigger realloc many times - struct fieldstat_tag shared_tag = TEST_SHARED_TAG; + struct field shared_tag = TEST_SHARED_TAG; for (int i = 0; i < registered_cube; i++) { shared_tag.value_longlong = i; - int cube_id = fieldstat_create_cube(instance, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10); EXPECT_EQ(cube_id, i); + fieldstat_register_counter(instance, cube_id, "counter"); } // try to use the cube - int metric_id = fieldstat_register_counter(instance, "counter"); for (int i = 0; i < registered_cube; i++) { - fieldstat_counter_incrby(instance, i, metric_id, &TEST_TAG_INT, 1, i); + fieldstat_counter_incrby(instance, i, 0, &TEST_FIELD_INT, 1, i); } for (int i = 0; i < registered_cube; i++) { long long result; - fieldstat_counter_get(instance, i, 0, &TEST_TAG_LIST_INT, &result); + fieldstat_counter_get(instance, i, &TEST_FIELD_LIST_INT, 0, &result); EXPECT_EQ(result, i); } @@ -143,18 +351,19 @@ TEST(test_register, register_many_cubes) TEST(test_register, add_many_tagged_cells) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - struct fieldstat_tag test_tag_long[100]; - for (int i = 0; i < 100; i++) { - test_tag_long[i] = TEST_TAG_INT; // will trigger realloc + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + size_t n_field = 1000; + struct field test_tag_long[n_field]; + for (size_t i = 0; i < n_field; i++) { + test_tag_long[i] = TEST_FIELD_INT; // will trigger realloc } - int metric_id = fieldstat_register_counter(instance, "counter"); - fieldstat_counter_incrby(instance, cube_id, metric_id, test_tag_long, 100, 10086); + int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + fieldstat_counter_incrby(instance, cube_id, metric_id, test_tag_long, n_field, 10086); long long result; - struct fieldstat_tag_list tag_list = {test_tag_long, 100}; - fieldstat_counter_get(instance, cube_id, metric_id, &tag_list, &result); + struct field_list tag_list = {test_tag_long, n_field}; + fieldstat_counter_get(instance, cube_id, &tag_list, metric_id, &result); EXPECT_EQ(result, 10086); fieldstat_free(instance); @@ -163,19 +372,19 @@ TEST(test_register, add_many_tagged_cells) TEST(test_register, add_long_tagged_cells) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - struct fieldstat_tag test_tag_long = TEST_TAG_STRING; + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + struct field test_tag_long = TEST_FIELD_STRING; char *long_string = (char *)malloc(5001); memset(long_string, 'a', 5001); long_string[5000] = '\0'; test_tag_long.value_str = long_string; - int metric_id = fieldstat_register_counter(instance, "counter"); + int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); fieldstat_counter_incrby(instance, cube_id, metric_id, &test_tag_long, 1, 10086); long long result; - struct fieldstat_tag_list tag_list = {&test_tag_long, 1}; - fieldstat_counter_get(instance, cube_id, metric_id, &tag_list, &result); + struct field_list tag_list = {&test_tag_long, 1}; + fieldstat_counter_get(instance, cube_id, &tag_list, metric_id, &result); EXPECT_EQ(result, 10086); fieldstat_free(instance); @@ -185,21 +394,21 @@ TEST(test_register, add_long_tagged_cells) TEST(test_register, register_many_metrics) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int metric_id = 0; for (int i = 0; i < 200; i++) { - metric_id = fieldstat_register_counter(instance, (std::string("counter ") + std::to_string(i)).c_str()); + metric_id = fieldstat_register_counter(instance, cube_id, (std::string("counter ") + std::to_string(i)).c_str()); EXPECT_EQ(metric_id, i); } for (int i = 0; i < 200; i++) { - EXPECT_EQ(fieldstat_counter_incrby(instance, cube_id, i, &TEST_TAG_INT, 1, 1), 0); + EXPECT_EQ(fieldstat_counter_incrby(instance, cube_id, i, &TEST_FIELD_INT, 1, 1), 0); } fieldstat_free(instance); } -TEST(test_register, dup_registered_info_from_empty_one) +TEST(test_register, fork_registered_info_from_empty_one) { struct fieldstat *instance = fieldstat_new(); @@ -214,16 +423,16 @@ TEST(test_register, dup_registered_info_from_empty_one) fieldstat_free(instance); } -TEST(test_register, dup_registered_info_with_cube_and_metric_without_cell) +TEST(test_register, fork_registered_info_with_cube_and_metric) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int metric_id = fieldstat_register_counter(instance, "counter"); - int metric_id2 = fieldstat_register_counter(instance, "counter2"); - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1); - int cube_id_del = fieldstat_create_cube(instance, &TEST_TAG_DOUBLE_collided, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int cube_id2 = fieldstat_create_cube(instance, &TEST_TAG_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - fieldstat_destroy_cube(instance, cube_id_del); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + int metric_id2 = fieldstat_register_counter(instance, cube_id, "counter2"); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, 1); + int cube_id_del = test_fieldstat_cube_create(instance, &TEST_FIELD_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + test_fieldstat_cube_create(instance, &TEST_FIELD_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + fieldstat_cube_destroy(instance, cube_id_del); struct fieldstat *dup = fieldstat_fork(instance); @@ -231,18 +440,25 @@ TEST(test_register, dup_registered_info_with_cube_and_metric_without_cell) int cube_num; fieldstat_get_cubes(dup, &cube_ids, &cube_num); EXPECT_EQ(cube_num, 2); - EXPECT_EQ(cube_ids[0], cube_id); - EXPECT_EQ(cube_ids[1], cube_id2); - free(cube_ids); - - EXPECT_STREQ(fieldstat_get_metric_name(dup, metric_id), "counter"); - EXPECT_STREQ(fieldstat_get_metric_name(dup, metric_id2), "counter2"); - - struct fieldstat_tag_list *tag_list = NULL; + struct field_list *tag_list = NULL; + + tag_list = fieldstat_cube_get_dimensions(dup, cube_ids[0]); + EXPECT_STREQ(tag_list->field[0].key, TEST_SHARED_TAG.key); + fieldstat_field_list_arr_free(tag_list, 1); size_t n_cell = 0; - fieldstat_get_cells_used_by_metric(dup, cube_id, metric_id, &tag_list, &n_cell); + fieldstat_cube_get_cells(dup, cube_ids[0], &tag_list, &n_cell); EXPECT_EQ(n_cell, 0); + tag_list = fieldstat_cube_get_dimensions(dup, cube_ids[1]); + EXPECT_STREQ(tag_list->field[0].key, TEST_FIELD_DOUBLE.key); + + fieldstat_field_list_arr_free(tag_list, 1); + + EXPECT_STREQ(fieldstat_get_metric_name(dup, cube_ids[0], metric_id), "counter"); + EXPECT_STREQ(fieldstat_get_metric_name(dup, cube_ids[0], metric_id2), "counter2"); + EXPECT_EQ(fieldstat_get_metric_name(dup, cube_ids[1], metric_id), nullptr); + + free(cube_ids); fieldstat_free(dup); fieldstat_free(instance); } @@ -252,16 +468,16 @@ TEST(test_register, unregister_cube_on_wrong_instance) struct fieldstat *instance = fieldstat_new(); struct fieldstat *instance_dst = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int cube_id2 = fieldstat_create_cube(instance, &TEST_TAG_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - int metric_id = fieldstat_register_counter(instance, "counter"); - int metric_id2 = fieldstat_register_counter(instance, "counter2"); - fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_TAG_INT, 1, 1); - fieldstat_counter_incrby(instance, cube_id2, metric_id2, &TEST_TAG_INT, 1, 1); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id2 = test_fieldstat_cube_create(instance, &TEST_FIELD_DOUBLE, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + int metric_id2 = fieldstat_register_counter(instance, cube_id2, "counter2"); + fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, 1); + fieldstat_counter_incrby(instance, cube_id2, metric_id2, &TEST_FIELD_INT, 1, 1); fieldstat_merge(instance_dst, instance); - fieldstat_destroy_cube(instance_dst, 0); + fieldstat_cube_destroy(instance_dst, 0); int *cube_ids; int cube_num; @@ -272,10 +488,10 @@ TEST(test_register, unregister_cube_on_wrong_instance) fieldstat_merge(instance_dst, instance); long long val_deleted_once; - fieldstat_counter_get(instance_dst, cube_id, metric_id, &TEST_TAG_LIST_INT, &val_deleted_once); + fieldstat_counter_get(instance_dst, cube_id, &TEST_FIELD_LIST_INT,metric_id, &val_deleted_once); EXPECT_EQ(val_deleted_once, 1); - long long val_merged_twice; - fieldstat_counter_get(instance_dst, cube_id2, metric_id2, &TEST_TAG_LIST_INT, &val_merged_twice); + long long val_merged_twice = -1; + fieldstat_counter_get(instance_dst, cube_id2, &TEST_FIELD_LIST_INT, metric_id2, &val_merged_twice); EXPECT_EQ(val_merged_twice, 2); fieldstat_free(instance); @@ -286,9 +502,9 @@ TEST(test_register, unregister_cube_on_wrong_instance) TEST(test_register, register_many_cells_on_unlimited_sized_cube) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 0); - int metric_id = fieldstat_register_counter(instance, "counter"); - struct fieldstat_tag test_tag = {"abc", TAG_INTEGER, {.value_longlong = 0}}; + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 0); + int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + struct field test_tag = {"abc", FIELD_VALUE_INTEGER, {.value_longlong = 0}}; for (int i = 0; i < 10000; i++) { test_tag.value_longlong = i; fieldstat_counter_incrby(instance, cube_id, metric_id, &test_tag, 1, 1); @@ -296,9 +512,9 @@ TEST(test_register, register_many_cells_on_unlimited_sized_cube) for (int i = 0; i < 10000; i++) { test_tag.value_longlong = i; - struct fieldstat_tag_list tag_list = {&test_tag, 1}; + struct field_list tag_list = {&test_tag, 1}; long long value; - fieldstat_counter_get(instance, cube_id, metric_id, &tag_list, &value); + fieldstat_counter_get(instance, cube_id, &tag_list, metric_id, &value); EXPECT_EQ(value, 1); } @@ -307,105 +523,129 @@ TEST(test_register, register_many_cells_on_unlimited_sized_cube) TEST(test_register, register_cube_twice) { struct fieldstat *instance = fieldstat_new(); - fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); - int cube_id2 = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); - EXPECT_EQ(cube_id2, FS_ERR_INVALID_KEY); + test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id2 = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + EXPECT_EQ(cube_id2, FS_ERR_DIMENSION_ALREADY_EXISTS); fieldstat_free(instance); } TEST(test_register, find_cube) { struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); int find_cube_id = fieldstat_find_cube(instance, &TEST_SHARED_TAG, 1); EXPECT_EQ(cube_id, find_cube_id); - int find_cube_id2 = fieldstat_find_cube(instance, &TEST_TAG_DOUBLE, 1); - EXPECT_EQ(find_cube_id2, FS_ERR_INVALID_KEY); + int find_cube_id2 = fieldstat_find_cube(instance, &TEST_FIELD_DOUBLE, 1); + EXPECT_EQ(find_cube_id2, FS_ERR_INVALID_DIMENSION); + + fieldstat_free(instance); +} + +TEST(test_register, set_sampling_error) { + struct fieldstat *instance = fieldstat_new(); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int ret = fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, 0); + EXPECT_EQ(ret, FS_ERR_INVALID_METRIC_ID); + int metric_id = fieldstat_register_hll(instance, cube_id, "hll", 5); + ret = fieldstat_cube_set_sampling(instance, cube_id, SAMPLING_MODE_TOPK, 10, metric_id); + EXPECT_EQ(ret, FS_ERR_INVALID_METRIC_TYPE); fieldstat_free(instance); } TEST(test_register, register_metric_twice) { struct fieldstat *instance = fieldstat_new(); - fieldstat_register_counter(instance, "counter"); - int metric_id2 = fieldstat_register_counter(instance, "counter"); - EXPECT_EQ(metric_id2, FS_ERR_INVALID_KEY); + int cube_id = test_fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + + fieldstat_register_counter(instance, cube_id, "counter"); + int metric_id2 = fieldstat_register_counter(instance, cube_id, "counter"); + EXPECT_EQ(metric_id2, FS_ERR_METRIC_NAME_ALREADY_EXISTS); + fieldstat_free(instance); +} + +TEST(test_register, add_on_uninitialized_cube) { + struct fieldstat *instance = fieldstat_new(); + int cube_id = fieldstat_cube_create(instance, &TEST_SHARED_TAG, 1); + int metric_id = fieldstat_register_counter(instance, cube_id, "counter"); + // fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, 1); + EXPECT_EQ(fieldstat_counter_incrby(instance, cube_id, metric_id, &TEST_FIELD_INT, 1, 1), FS_ERR_CUBE_SAMPLING_NOT_INITIALIZED); + fieldstat_free(instance); } TEST(calibrate, target_one_more_metric) { struct fieldstat *master = fieldstat_new(); - int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - fieldstat_register_counter(master, "counter"); + int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + fieldstat_register_counter(master, cube_id, "counter"); struct fieldstat *target = fieldstat_fork(master); - EXPECT_EQ(fieldstat_register_counter(target, "counter2"), 1); + EXPECT_EQ(fieldstat_register_counter(target, cube_id, "counter2"), 1); fieldstat_calibrate(master, target); EXPECT_EQ(test_get_max_metric_id(target), 0); - EXPECT_STREQ(fieldstat_get_metric_name(target, 0), "counter"); - struct fieldstat_tag_list *tag_list = fieldstat_get_shared_tags(target, cube_id); - EXPECT_STREQ(tag_list->tag[0].key, TEST_SHARED_TAG.key); + EXPECT_STREQ(fieldstat_get_metric_name(target, cube_id, 0), "counter"); + struct field_list *tag_list = fieldstat_cube_get_dimensions(target, cube_id); + EXPECT_STREQ(tag_list->field[0].key, TEST_SHARED_TAG.key); fieldstat_free(master); fieldstat_free(target); - fieldstat_tag_list_arr_free(tag_list, 1); + fieldstat_field_list_arr_free(tag_list, 1); } TEST(calibrate, master_one_more_metric) { struct fieldstat *master = fieldstat_new(); - int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - fieldstat_register_counter(master, "counter"); + int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + fieldstat_register_counter(master, cube_id, "counter"); struct fieldstat *target = fieldstat_fork(master); - EXPECT_EQ(fieldstat_register_counter(master, "counter2"), 1); + EXPECT_EQ(fieldstat_register_counter(master, cube_id, "counter2"), 1); fieldstat_calibrate(master, target); EXPECT_EQ(test_get_max_metric_id(target), 1); - EXPECT_STREQ(fieldstat_get_metric_name(target, 0), "counter"); - EXPECT_STREQ(fieldstat_get_metric_name(target, 1), "counter2"); - struct fieldstat_tag_list *tag_list = fieldstat_get_shared_tags(target, cube_id); - EXPECT_STREQ(tag_list->tag[0].key, TEST_SHARED_TAG.key); + EXPECT_STREQ(fieldstat_get_metric_name(target, cube_id,0), "counter"); + EXPECT_STREQ(fieldstat_get_metric_name(target, cube_id,1), "counter2"); + struct field_list *tag_list = fieldstat_cube_get_dimensions(target, cube_id); + EXPECT_STREQ(tag_list->field[0].key, TEST_SHARED_TAG.key); fieldstat_free(master); fieldstat_free(target); - fieldstat_tag_list_arr_free(tag_list, 1); + fieldstat_field_list_arr_free(tag_list, 1); } // different metric on same cube TEST(calibrate, different_metric) { struct fieldstat *master = fieldstat_new(); - int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - fieldstat_register_counter(master, "counter"); + int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + fieldstat_register_counter(master, cube_id, "counter"); struct fieldstat *target = fieldstat_fork(master); - EXPECT_EQ(fieldstat_register_counter(target, "counter2"), 1); - EXPECT_EQ(fieldstat_register_counter(master, "hi i am master new"), 1); + EXPECT_EQ(fieldstat_register_counter(target, cube_id, "counter2"), 1); + EXPECT_EQ(fieldstat_register_counter(master, cube_id, "hi i am master new"), 1); fieldstat_calibrate(master, target); EXPECT_EQ(test_get_max_metric_id(target), 1); - EXPECT_STREQ(fieldstat_get_metric_name(target, 0), "counter"); - EXPECT_STREQ(fieldstat_get_metric_name(target, 1), "hi i am master new"); - struct fieldstat_tag_list *tag_list = fieldstat_get_shared_tags(target, cube_id); - EXPECT_STREQ(tag_list->tag[0].key, TEST_SHARED_TAG.key); + EXPECT_STREQ(fieldstat_get_metric_name(target, cube_id, 0), "counter"); + EXPECT_STREQ(fieldstat_get_metric_name(target, cube_id, 1), "hi i am master new"); + struct field_list *tag_list = fieldstat_cube_get_dimensions(target, cube_id); + EXPECT_STREQ(tag_list->field[0].key, TEST_SHARED_TAG.key); fieldstat_free(master); fieldstat_free(target); - fieldstat_tag_list_arr_free(tag_list, 1); + fieldstat_field_list_arr_free(tag_list, 1); } TEST(calibrate, target_more_cube) { struct fieldstat *master = fieldstat_new(); - int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - fieldstat_register_counter(master, "counter"); + int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + fieldstat_register_counter(master, cube_id, "counter"); struct fieldstat *target = fieldstat_fork(master); - int cube_id2 = fieldstat_create_cube(target, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id2 = test_fieldstat_cube_create(target, &TEST_FIELD_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10); EXPECT_EQ(cube_id2, 1); - fieldstat_register_counter(target, "counter2"); + fieldstat_register_counter(target, cube_id, "counter2"); fieldstat_calibrate(master, target); @@ -417,24 +657,24 @@ TEST(calibrate, target_more_cube) free(cube_ids); EXPECT_EQ(test_get_max_metric_id(target), 0); - EXPECT_STREQ(fieldstat_get_metric_name(target, 0), "counter"); - struct fieldstat_tag_list *tag_list = fieldstat_get_shared_tags(target, cube_id); - EXPECT_STREQ(tag_list->tag[0].key, TEST_SHARED_TAG.key); + EXPECT_STREQ(fieldstat_get_metric_name(target, cube_id, 0), "counter"); + struct field_list *tag_list = fieldstat_cube_get_dimensions(target, cube_id); + EXPECT_STREQ(tag_list->field[0].key, TEST_SHARED_TAG.key); EXPECT_EQ(fieldstat_find_cube(target, &TEST_SHARED_TAG, 1), 0); - EXPECT_EQ(fieldstat_find_cube(target, &TEST_TAG_STRING, 1), FS_ERR_INVALID_KEY); + EXPECT_EQ(fieldstat_find_cube(target, &TEST_FIELD_STRING, 1), FS_ERR_INVALID_DIMENSION); fieldstat_free(master); fieldstat_free(target); - fieldstat_tag_list_arr_free(tag_list, 1); + fieldstat_field_list_arr_free(tag_list, 1); } TEST(calibrate, master_more_cube) { struct fieldstat *master = fieldstat_new(); - int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); struct fieldstat *target = fieldstat_fork(master); - int cube_id2 = fieldstat_create_cube(master, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id2 = test_fieldstat_cube_create(master, &TEST_FIELD_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10); fieldstat_calibrate(master, target); @@ -446,57 +686,55 @@ TEST(calibrate, master_more_cube) EXPECT_EQ(cube_ids[1], cube_id2); free(cube_ids); - struct fieldstat_tag_list *tag_list = fieldstat_get_shared_tags(target, cube_id); - EXPECT_STREQ(tag_list->tag[0].key, TEST_SHARED_TAG.key); - struct fieldstat_tag_list *tag_list2 = fieldstat_get_shared_tags(target, cube_id2); - EXPECT_STREQ(tag_list2->tag[0].key, TEST_TAG_STRING.key); + struct field_list *tag_list = fieldstat_cube_get_dimensions(target, cube_id); + EXPECT_STREQ(tag_list->field[0].key, TEST_SHARED_TAG.key); + struct field_list *tag_list2 = fieldstat_cube_get_dimensions(target, cube_id2); + EXPECT_STREQ(tag_list2->field[0].key, TEST_FIELD_STRING.key); - EXPECT_EQ(fieldstat_find_cube(target, &TEST_TAG_STRING, 1), 1); + EXPECT_EQ(fieldstat_find_cube(target, &TEST_FIELD_STRING, 1), 1); EXPECT_EQ(fieldstat_find_cube(target, &TEST_SHARED_TAG, 1), 0); fieldstat_free(master); fieldstat_free(target); - fieldstat_tag_list_arr_free(tag_list, 1); - fieldstat_tag_list_arr_free(tag_list2, 1); + fieldstat_field_list_arr_free(tag_list, 1); + fieldstat_field_list_arr_free(tag_list2, 1); } TEST(calibrate, master_change_cube) { struct fieldstat *master = fieldstat_new(); - int cube_id = fieldstat_create_cube(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - fieldstat_register_counter(master, "counter"); + int cube_id = test_fieldstat_cube_create(master, &TEST_SHARED_TAG, 1, SAMPLING_MODE_COMPREHENSIVE, 10); struct fieldstat *target = fieldstat_fork(master); - fieldstat_destroy_cube(master, cube_id); - fieldstat_create_cube(master, &TEST_TAG_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + fieldstat_cube_destroy(master, cube_id); + test_fieldstat_cube_create(master, &TEST_FIELD_STRING, 1, SAMPLING_MODE_COMPREHENSIVE, 10); fieldstat_calibrate(master, target); - struct fieldstat_tag_list *tag_list = fieldstat_get_shared_tags(target, cube_id); - EXPECT_STREQ(tag_list->tag[0].key, TEST_TAG_STRING.key); + struct field_list *tag_list = fieldstat_cube_get_dimensions(target, cube_id); + EXPECT_STREQ(tag_list->field[0].key, TEST_FIELD_STRING.key); - EXPECT_EQ(fieldstat_find_cube(target, &TEST_TAG_STRING, 1), 0); - EXPECT_EQ(fieldstat_get_cube_version(target, cube_id), 1); + EXPECT_EQ(fieldstat_find_cube(target, &TEST_FIELD_STRING, 1), 0); fieldstat_free(master); fieldstat_free(target); - fieldstat_tag_list_arr_free(tag_list, 1); + fieldstat_field_list_arr_free(tag_list, 1); } TEST(calibrate, master_many_cube) { struct fieldstat *master = fieldstat_new(); int registered_cube = 3000; // will trigger realloc 1 times - struct fieldstat_tag shared_tag = TEST_SHARED_TAG; + struct field shared_tag = TEST_SHARED_TAG; for (int i = 0; i < registered_cube; i++) { shared_tag.value_longlong = i; - int cube_id = fieldstat_create_cube(master, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(master, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10); EXPECT_EQ(cube_id, i); } struct fieldstat *target = fieldstat_fork(master); int new_registered = 5000; for (int i = 0; i < new_registered; i++) { shared_tag.value_longlong = i + registered_cube; - int cube_id = fieldstat_create_cube(master, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10); + int cube_id = test_fieldstat_cube_create(master, &shared_tag, 1, SAMPLING_MODE_COMPREHENSIVE, 10); EXPECT_EQ(cube_id, i + registered_cube); } @@ -517,32 +755,6 @@ TEST(calibrate, master_many_cube) fieldstat_free(target); } -TEST(test_register, get_cube_mode) -{ - // int fieldstat_get_cube_mode(const struct fieldstat *instance, int cube_id, enum sampling_mode *mode, int *primary_metric_id) - - struct fieldstat *instance = fieldstat_new(); - int cube_id = fieldstat_create_cube(instance, &TEST_SHARED_TAG, 1, SAMPLING_MODE_TOPK, 10); - fieldstat_register_counter(instance, "counter"); - fieldstat_register_counter(instance, "counter2"); - fieldstat_cube_set_primary_metric(instance, cube_id, 1); - int cube_id2 = fieldstat_create_cube(instance, &TEST_TAG_INT, 1, SAMPLING_MODE_COMPREHENSIVE, 10); - - enum sampling_mode mode; - int primary_metric_id; - EXPECT_EQ(fieldstat_get_cube_mode(instance, cube_id, &mode, &primary_metric_id), 0); - EXPECT_EQ(mode, SAMPLING_MODE_TOPK); - EXPECT_EQ(primary_metric_id, 1); - - EXPECT_EQ(fieldstat_get_cube_mode(instance, cube_id2, &mode, &primary_metric_id), 0); - EXPECT_EQ(mode, SAMPLING_MODE_COMPREHENSIVE); - EXPECT_EQ(primary_metric_id, -1); - - EXPECT_EQ(fieldstat_get_cube_mode(instance, 100, &mode, &primary_metric_id), FS_ERR_INVALID_CUBE_ID); - - fieldstat_free(instance); -} - // issue： https://jira.geedge.net/browse/TSG-20140 // 流程： // calibrate调用前： @@ -552,7 +764,7 @@ TEST(test_register, get_cube_mode) // calibrate调用时，依次比较每个cube id，id:0时，将master[0] 同步,同步时，会先删除A,然后添加B' // replica:B' B (B' B) -// calibrate 继续，同步master[1]，replica[1]不存在，创建新的cube。此时，不仅从id->cube 的数组中删除掉B，且从cube tag->id 的字典中删除掉B。可能会错误的删除掉B'，导致两个map的对应不一致。 +// calibrate 继续，同步master[1]，replica[1]不存在，创建新的cube。此时，不仅从id->cube 的数组中删除掉B，且从cube field->id 的字典中删除掉B。可能会错误的删除掉B'，导致两个map的对应不一致。 // replica:B' / (B ) // 例如：find_cube(B)-> 返回1。 @@ -560,15 +772,15 @@ TEST(test_register, get_cube_mode) TEST(calibrate, issue_calibrate_wrong) { struct fieldstat *master = fieldstat_new(); - const struct fieldstat_tag *tag_A = &TEST_SHARED_TAG; - const struct fieldstat_tag *tag_B = &TEST_TAG_INT; - int cube_idA = fieldstat_create_cube(master, tag_A, 1, SAMPLING_MODE_COMPREHENSIVE, 1); - int cube_idB = fieldstat_create_cube(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1); + const struct field *tag_A = &TEST_SHARED_TAG; + const struct field *tag_B = &TEST_FIELD_INT; + int cube_idA = test_fieldstat_cube_create(master, tag_A, 1, SAMPLING_MODE_COMPREHENSIVE, 1); + int cube_idB = test_fieldstat_cube_create(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1); struct fieldstat *target = fieldstat_fork(master); - EXPECT_EQ(fieldstat_destroy_cube(master, cube_idA), FS_OK); - EXPECT_EQ(fieldstat_destroy_cube(master, cube_idB), FS_OK); - int cube_idBa = fieldstat_create_cube(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1); + EXPECT_EQ(fieldstat_cube_destroy(master, cube_idA), FS_OK); + EXPECT_EQ(fieldstat_cube_destroy(master, cube_idB), FS_OK); + int cube_idBa = test_fieldstat_cube_create(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1); EXPECT_EQ(cube_idBa, 0); fieldstat_calibrate(master, target); @@ -577,30 +789,28 @@ TEST(calibrate, issue_calibrate_wrong) int n_cubes = 0; fieldstat_get_cubes(target, &cubes_id_ret, &n_cubes); EXPECT_EQ(n_cubes, 1); - EXPECT_EQ(cubes_id_ret[0], 0); - free(cubes_id_ret); + EXPECT_EQ(fieldstat_find_cube(target, tag_A, 1), FS_ERR_INVALID_DIMENSION); + EXPECT_EQ(fieldstat_find_cube(target, tag_B, 1), cubes_id_ret[0]); - EXPECT_EQ(fieldstat_find_cube(target, tag_A, 1), FS_ERR_INVALID_KEY); - EXPECT_EQ(fieldstat_find_cube(target, tag_B, 1), 0); - - struct fieldstat_tag_list *tag_list = fieldstat_get_shared_tags(target, 0); - EXPECT_STREQ(tag_list->tag[0].key, tag_B->key); - fieldstat_tag_list_arr_free(tag_list, 1); + struct field_list *tag_list = fieldstat_cube_get_dimensions(target, cubes_id_ret[0]); + EXPECT_STREQ(tag_list->field[0].key, tag_B->key); + fieldstat_field_list_arr_free(tag_list, 1); fieldstat_free(master); fieldstat_free(target); + free(cubes_id_ret); } TEST(calibrate, delete_first_cube) { struct fieldstat *master = fieldstat_new(); - const struct fieldstat_tag *tag_A = &TEST_SHARED_TAG; - const struct fieldstat_tag *tag_B = &TEST_TAG_INT; - int cube_idA = fieldstat_create_cube(master, tag_A, 1, SAMPLING_MODE_COMPREHENSIVE, 1); - int cube_idB = fieldstat_create_cube(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1); + const struct field *tag_A = &TEST_SHARED_TAG; + const struct field *tag_B = &TEST_FIELD_INT; + int cube_idA = test_fieldstat_cube_create(master, tag_A, 1, SAMPLING_MODE_COMPREHENSIVE, 1); + int cube_idB = test_fieldstat_cube_create(master, tag_B, 1, SAMPLING_MODE_COMPREHENSIVE, 1); struct fieldstat *target = fieldstat_fork(master); - fieldstat_destroy_cube(master, cube_idA); + fieldstat_cube_destroy(master, cube_idA); fieldstat_calibrate(master, target); @@ -611,12 +821,12 @@ TEST(calibrate, delete_first_cube) EXPECT_EQ(cubes_id_ret[0], cube_idB); free(cubes_id_ret); - EXPECT_EQ(fieldstat_find_cube(target, tag_A, 1), FS_ERR_INVALID_KEY); + EXPECT_EQ(fieldstat_find_cube(target, tag_A, 1), FS_ERR_INVALID_DIMENSION); EXPECT_EQ(fieldstat_find_cube(target, tag_B, 1), 1); - struct fieldstat_tag_list *tag_list = fieldstat_get_shared_tags(target, 1); - EXPECT_STREQ(tag_list->tag[0].key, tag_B->key); - fieldstat_tag_list_arr_free(tag_list, 1); + struct field_list *tag_list = fieldstat_cube_get_dimensions(target, 1); + EXPECT_STREQ(tag_list->field[0].key, tag_B->key); + fieldstat_field_list_arr_free(tag_list, 1); fieldstat_free(master); fieldstat_free(target); @@ -625,5 +835,6 @@ TEST(calibrate, delete_first_cube) int main(int argc, char *argv[]) { testing::InitGoogleTest(&argc, argv); + // testing::GTEST_FLAG(filter) = "test_register.reset_and_new_cell_spreadsketch"; return RUN_ALL_TESTS(); }
\ No newline at end of file diff --git a/test/test_write_json_file.cpp b/test/test_write_json_file.cpp index 6d6131c..1dba94b 100644 --- a/test/test_write_json_file.cpp +++ b/test/test_write_json_file.cpp @@ -22,23 +22,23 @@ extern "C" { extern char *fs_easy_output_to_json(struct fieldstat_easy *fs, const struct timeval *timestamp, const struct timeval *timestamp_delta); } -static struct fieldstat_easy *get_hist_fieldstat_easy() +[[gnu::unused]] static struct fieldstat_easy *get_hist_fieldstat_easy() { - struct fieldstat_tag global_tags[2]; - struct fieldstat_tag cell_tags[2]; + struct field global_tags[2]; + struct field cell_tags[2]; global_tags[0].key = "rule_id"; - global_tags[0].type = TAG_INTEGER; + global_tags[0].type = FIELD_VALUE_INTEGER; global_tags[0].value_longlong = 1; global_tags[1].key = "action"; - global_tags[1].type = TAG_CSTRING; + global_tags[1].type = FIELD_VALUE_CSTRING; global_tags[1].value_str = "deny"; cell_tags[0].key = "thread_id"; - cell_tags[0].type = TAG_INTEGER; + cell_tags[0].type = FIELD_VALUE_INTEGER; cell_tags[0].value_longlong = 1; cell_tags[1].key = "hit_rate"; - cell_tags[1].type = TAG_DOUBLE; + cell_tags[1].type = FIELD_VALUE_DOUBLE; cell_tags[1].value_double = 1.1; const char *hist_names[] = {"list_num", "max_wt_ms", "ivt_nx_itv_ms", @@ -58,24 +58,28 @@ static struct fieldstat_easy *get_hist_fieldstat_easy() return fse; } -static struct fieldstat *get_hll_fieldsstat() +int test_fieldstat_cube_create(struct fieldstat *instance, const struct field *dimensions, size_t n_dimensions, enum sampling_mode mode, int k) { - struct fieldstat_tag shared_tags[1]; + assert(mode == SAMPLING_MODE_COMPREHENSIVE); + int ret = fieldstat_cube_create(instance, dimensions, n_dimensions); + fieldstat_cube_set_sampling(instance, ret, mode, k, 0); + return ret; +} + +static void write_hll(struct fieldstat *instance) { + struct field shared_tags[1]; shared_tags[0].key = "rule_id"; - shared_tags[0].type = TAG_INTEGER; + shared_tags[0].type = FIELD_VALUE_INTEGER; shared_tags[0].value_longlong = 1; const char *hll_name[] = {"external_ip", "internal_ip", "acc_ip"}; - - struct fieldstat *instance = fieldstat_new(); - EXPECT_NE(nullptr, instance); - int cube_id = fieldstat_create_cube(instance, shared_tags, 1, + int cube_id = test_fieldstat_cube_create(instance, shared_tags, 1, SAMPLING_MODE_COMPREHENSIVE, 100); for(unsigned int i = 0; i < sizeof(hll_name) / sizeof(hll_name[0]); i++) { - int hll_id = fieldstat_register_hll(instance, hll_name[i], 5); + int hll_id = fieldstat_register_hll(instance, cube_id, hll_name[i], 5); for(int j = 0; j < 100; j++) { @@ -85,30 +89,25 @@ static struct fieldstat *get_hll_fieldsstat() int ret = fieldstat_hll_add(instance, cube_id, hll_id, NULL, 0, ip_str, strlen(ip_str)); EXPECT_EQ(0, ret); } - } - - return instance; } - -static struct fieldstat *get_hist_fieldstat() -{ - struct fieldstat_tag shared_tags[2]; - struct fieldstat_tag cell_tags[2]; +void write_histogram(struct fieldstat *instance) { + struct field shared_tags[2]; + struct field cell_tags[2]; shared_tags[0].key = "rule_id"; - shared_tags[0].type = TAG_INTEGER; + shared_tags[0].type = FIELD_VALUE_INTEGER; shared_tags[0].value_longlong = 1; shared_tags[1].key = "action"; - shared_tags[1].type = TAG_CSTRING; + shared_tags[1].type = FIELD_VALUE_CSTRING; shared_tags[1].value_str = "deny"; cell_tags[0].key = "thread_id"; - cell_tags[0].type = TAG_INTEGER; + cell_tags[0].type = FIELD_VALUE_INTEGER; cell_tags[0].value_longlong = 1; cell_tags[1].key = "hit_rate"; - cell_tags[1].type = TAG_DOUBLE; + cell_tags[1].type = FIELD_VALUE_DOUBLE; cell_tags[1].value_double = 1.1; const char *hist_names[] = {"list_num", "max_wt_ms", "ivt_nx_itv_ms", @@ -116,35 +115,30 @@ static struct fieldstat *get_hist_fieldstat() "bye/udp", "oth_mtd/udp"}; - struct fieldstat *instance = fieldstat_new(); - EXPECT_NE(nullptr, instance); - int cube_id = fieldstat_create_cube(instance, shared_tags, 2, + int cube_id = test_fieldstat_cube_create(instance, shared_tags, 2, SAMPLING_MODE_COMPREHENSIVE, 100); - EXPECT_EQ(0, cube_id); for(unsigned int i = 0; i < sizeof(hist_names)/sizeof(hist_names[0]); i++) { - int hist_id = fieldstat_register_hist(instance, hist_names[i], 1, 600000, 3); + int hist_id = fieldstat_register_histogram(instance, cube_id, hist_names[i], 1, 600000, 3); for(int j = 0; j < 100; j++) { - fieldstat_hist_record(instance, cube_id, hist_id, cell_tags, 2, i*100 + j); + fieldstat_histogram_record(instance, cube_id, hist_id, cell_tags, 2, i*100 + j); } } - - return instance; } // static struct fieldstat_easy *get_table_fieldstat_easy() // { -// struct fieldstat_tag global_tags[2]; -// struct fieldstat_tag cell_tags; +// struct field global_tags[2]; +// struct field cell_tags; // global_tags[0].key = "policy_id"; -// global_tags[0].type = TAG_INTEGER; +// global_tags[0].type = FIELD_VALUE_INTEGER; // global_tags[0].value_longlong = 1; // global_tags[1].key = "quanlity"; -// global_tags[1].type = TAG_DOUBLE; +// global_tags[1].type = FIELD_VALUE_DOUBLE; // global_tags[1].value_double = 0.5; // const char *cell_tag_value[] = { @@ -154,7 +148,7 @@ static struct fieldstat *get_hist_fieldstat() // "STATISTICS-RULE-METRIC", "OBJECT-STATISTICS-METRIC"}; // cell_tags.key = "send_log"; -// cell_tags.type = TAG_CSTRING; +// cell_tags.type = FIELD_VALUE_CSTRING; // cell_tags.value_str = "true"; // struct fieldstat_easy *fse = fieldstat_easy_new(N_THREAD, NULL, global_tags, 2); @@ -179,14 +173,13 @@ void fieldstat_easy_to_file(struct fieldstat_easy *fse, const char *file_path) fieldstat_easy_free(fse); // only by free to stop exporting } -static struct fieldstat *get_table_fieldstat() -{ - struct fieldstat_tag shared_tags[2]; +void write_table(struct fieldstat *instance) { + struct field shared_tags[2]; shared_tags[0].key = "policy_id"; - shared_tags[0].type = TAG_INTEGER; + shared_tags[0].type = FIELD_VALUE_INTEGER; shared_tags[0].value_longlong = 1; shared_tags[1].key = "quanlity"; - shared_tags[1].type = TAG_DOUBLE; + shared_tags[1].type = FIELD_VALUE_DOUBLE; shared_tags[1].value_double = 0.5; const char *cell_tag_value[] = { @@ -195,21 +188,17 @@ static struct fieldstat *get_table_fieldstat() "GTPC-RECORD", "BGP-RECORD", "PROXY-EVENT", "DOS-SKETCH-RECORD", "STATISTICS-RULE-METRIC", "OBJECT-STATISTICS-METRIC"}; - struct fieldstat_tag cell_tags; + struct field cell_tags; cell_tags.key = "send_log"; - cell_tags.type = TAG_CSTRING; + cell_tags.type = FIELD_VALUE_CSTRING; cell_tags.value_str = "true"; - struct fieldstat *instance = fieldstat_new(); - EXPECT_NE(nullptr, instance); - - int cube_id = fieldstat_create_cube(instance, shared_tags, 2, + int cube_id = test_fieldstat_cube_create(instance, shared_tags, 2, SAMPLING_MODE_COMPREHENSIVE, 100); - EXPECT_EQ(0, cube_id); - int counter_id_0 = fieldstat_register_counter(instance, "T_success_log"); + int counter_id_0 = fieldstat_register_counter(instance, cube_id, "T_success_log"); - int counter_id_1 = fieldstat_register_counter(instance, "T_fail_log"); + int counter_id_1 = fieldstat_register_counter(instance, cube_id, "T_fail_log"); for(unsigned int i = 0; i < sizeof(cell_tag_value)/sizeof(cell_tag_value[0]); i++) { @@ -218,8 +207,6 @@ static struct fieldstat *get_table_fieldstat() if(i < 5) fieldstat_counter_incrby(instance, cube_id, counter_id_1, &cell_tags, 1, 2); } - - return instance; } static int write_json_to_file(const char *filename, char *json_str) @@ -239,13 +226,11 @@ TEST(ExporterLocal, TableBuild) { struct timeval current = {100, 10000}; struct fieldstat *merger = fieldstat_new(); - struct fieldstat *hll = get_hll_fieldsstat(); - struct fieldstat *hist = get_hist_fieldstat(); - struct fieldstat *table = get_table_fieldstat(); - fieldstat_merge(merger, hll); - fieldstat_merge(merger, hist); - fieldstat_merge(merger, table); + write_hll(merger); + write_histogram(merger); + write_table(merger); + struct fieldstat_json_exporter *exporter = fieldstat_json_exporter_new(); fieldstat_json_exporter_enable_delta(exporter); char *str_json = fieldstat_json_exporter_export(exporter, merger, ¤t); diff --git a/test/unit_test_cell_manager.cpp b/test/unit_test_cell_manager.cpp deleted file mode 100644 index 67c1033..0000000 --- a/test/unit_test_cell_manager.cpp +++ /dev/null @@ -1,733 +0,0 @@ - -#include <gtest/gtest.h> -#include <map> -#include <set> -#include <string> -#include <vector> -#include <algorithm> - - -#include "fieldstat.h" -#include "tags/cell_manager.h" -#include "utils.hpp" -#include "tags/my_ut_hash.h" - -using namespace std; - -struct tag_hash_key *test_gen_tag_key(const char *key, int value) -{ - struct fieldstat_tag tag = { - .key = key, - .type = TAG_CSTRING, - {.value_str = strdup(to_string(value).c_str())}, - }; - - struct tag_hash_key *tag_key = (struct tag_hash_key *)malloc(sizeof(struct tag_hash_key)); - tag_hash_key_init_with_fieldstat_tag(tag_key, &tag, 1, true); - - free((void *)tag.value_str); - - return tag_key; -} - -struct Fieldstat_tag_list_wrapper *test_key_tag_to_wrapper(const struct tag_hash_key *key) -{ - struct fieldstat_tag *tag; - size_t n_out; - tag_hash_key_convert_to_fieldstat_tag(key, &tag, &n_out); - struct fieldstat_tag_list tag_list; - tag_list.tag = tag; - tag_list.n_tag = n_out; - struct Fieldstat_tag_list_wrapper *wrapper = new Fieldstat_tag_list_wrapper(&tag_list); - - for (size_t i = 0; i < n_out; i++) - { - if (tag[i].type == TAG_CSTRING) - free((void *)tag[i].value_str); - free((void *)tag[i].key); - } - free(tag); - - return wrapper; -} - -double cal_accuracy_with_tags(const vector<struct tag_hash_key *> &expected_keys, const vector<struct tag_hash_key *> &test_result) { - unordered_map<string, int> countMap; - for (size_t i = 0; i < expected_keys.size(); i++) { - struct Fieldstat_tag_list_wrapper *wrapper = test_key_tag_to_wrapper(expected_keys[i]); - string key = wrapper->to_string(); - countMap[key]++; - delete wrapper; - } - vector<struct Fieldstat_tag_list_wrapper *> test_result_wrapper; - - for (size_t i = 0; i < test_result.size(); i++) { - struct Fieldstat_tag_list_wrapper *wrapper = test_key_tag_to_wrapper(test_result[i]); - test_result_wrapper.push_back(wrapper); - } - - double ret = test_cal_topk_accuracy(test_result_wrapper, countMap); - - for (size_t i = 0; i < test_result.size(); i++) { - delete test_result_wrapper[i]; - } - return ret; -} - -vector<tag_hash_key *> test_query_cell_manager_content(const struct cell_manager *cm) -{ - int ret_len; - const struct tag_hash_key **dump_ret = cell_manager_dump(cm, &ret_len); - vector<tag_hash_key *> test_result; - for (int i = 0; i < ret_len; i++) { - if (dump_ret[i] == NULL) { - continue; - } - - test_result.push_back((struct tag_hash_key *)dump_ret[i]); - } - - return test_result; -} - -TEST(unit_test_cell_manager, topk_simple_add) -{ - struct cell_manager *cm = cell_manager_new(SAMPLING_MODE_TOPK, 10); - const int TEST_ROUND = 10; - - vector<struct tag_hash_key *> keys; - for (int i = 0; i < TEST_ROUND; i++) - { - struct tag_hash_key *key = test_gen_tag_key("key", i); - keys.push_back(key); - } - - int pop_dummy; - - for (int i = 0; i < TEST_ROUND; i++) { - cell_manager_add_cell_topk(cm, keys[i], 1, &pop_dummy); - } - - vector<tag_hash_key *> test_result = test_query_cell_manager_content(cm); - EXPECT_EQ(test_result.size(), 10); - double accuracy = cal_accuracy_with_tags(keys, test_result); - EXPECT_NEAR(accuracy, 1.0, 0.01); - - cell_manager_free(cm); - for (int i = 0; i < TEST_ROUND; i++) { - tag_hash_key_free(keys[i]); - } -} - -TEST(unit_test_cell_manager, topk_add_pop) -{ - struct cell_manager *cm = cell_manager_new(SAMPLING_MODE_TOPK, 10); - const int TEST_ROUND = 11; - - vector<struct tag_hash_key *> keys; - for (int i = 0; i < TEST_ROUND; i++) - { - struct tag_hash_key *key = test_gen_tag_key("key", i); - keys.push_back(key); - } - - int pop_dummy; - for (int i = 0; i < TEST_ROUND - 1; i++) { - cell_manager_add_cell_topk(cm, keys[i], 1, &pop_dummy); - } - cell_manager_add_cell_topk(cm, keys[TEST_ROUND - 1], 100, &pop_dummy); - - vector<tag_hash_key *> test_result = test_query_cell_manager_content(cm); - EXPECT_EQ(test_result.size(), 10); - double accuracy = cal_accuracy_with_tags(keys, test_result); - EXPECT_NEAR(accuracy, 1.0, 0.01); - - EXPECT_EQ(cell_manager_get_count_by_tag(cm, keys[TEST_ROUND - 1]), 100); - - cell_manager_free(cm); - for (int i = 0; i < TEST_ROUND; i++) { - tag_hash_key_free(keys[i]); - } -} - -TEST(unit_test_cell_manager, topk_add_twice) -{ - struct cell_manager *cm = cell_manager_new(SAMPLING_MODE_TOPK, 10); - const int TEST_ROUND = 10; - - vector<struct tag_hash_key *> keys; - for (int i = 0; i < TEST_ROUND; i++) - { - struct tag_hash_key *key = test_gen_tag_key("key", i); - keys.push_back(key); - } - - int pop_dummy; - for (int i = 0; i < TEST_ROUND; i++) { - cell_manager_add_cell_topk(cm, keys[i], 1, &pop_dummy); - } - cell_manager_add_cell_topk(cm, keys[TEST_ROUND - 1], 100, &pop_dummy); - - vector<tag_hash_key *> test_result = test_query_cell_manager_content(cm); - EXPECT_EQ(test_result.size(), 10); - double accuracy = cal_accuracy_with_tags(keys, test_result); - EXPECT_NEAR(accuracy, 1.0, 0.01); - - cell_manager_free(cm); - for (int i = 0; i < TEST_ROUND; i++) { - tag_hash_key_free(keys[i]); - } -} - -TEST(unit_test_cell_manager, topk_add_and_query_accuracy) -{ - struct cell_manager *cm = cell_manager_new(SAMPLING_MODE_TOPK, 10); - const int TEST_ROUND = 10000; - - vector<struct tag_hash_key *> keys; - for (int i = 0; i < TEST_ROUND; i++) - { - if (rand()) { - struct tag_hash_key *key = test_gen_tag_key("key", rand() % 10); - keys.push_back(key); - } else { - struct tag_hash_key *key = test_gen_tag_key("key", rand() % 1000); - keys.push_back(key); - } - } - - int pop_dummy; - for (int i = 0; i < TEST_ROUND; i++) { - cell_manager_add_cell_topk(cm, keys[i], 1, &pop_dummy); - } - - vector<tag_hash_key *> test_result = test_query_cell_manager_content(cm); - EXPECT_EQ(test_result.size(), 10); - double accuracy = cal_accuracy_with_tags(keys, test_result); - EXPECT_NEAR(accuracy, 1.0, 0.01); - - cell_manager_free(cm); - for (int i = 0; i < TEST_ROUND; i++) { - tag_hash_key_free(keys[i]); - } -} - -TEST(unit_test_cell_manager, merge_topk_given_K_large_enough) -{ - struct cell_manager *cm1 = cell_manager_new(SAMPLING_MODE_TOPK, 10); - struct cell_manager *cm2 = cell_manager_new(SAMPLING_MODE_TOPK, 10); - - vector<struct tag_hash_key *> keys; - keys.push_back(test_gen_tag_key("key_share", 1)); - keys.push_back(test_gen_tag_key("key_1", 1)); - keys.push_back(test_gen_tag_key("key_1", 2)); - keys.push_back(test_gen_tag_key("key_share", 1)); - keys.push_back(test_gen_tag_key("key_2", 1)); - - int pop_dummy; - int cell_id_1[3]; - int cell_id_2[2]; - - for (size_t i = 0; i < 3; i++) { - cell_id_1[i] = cell_manager_add_cell_topk(cm1, keys[i], 1, &pop_dummy); - } - for (size_t i = 3; i < 5; i++) { - cell_id_2[i - 3] = cell_manager_add_cell_topk(cm2, keys[i], 1, &pop_dummy); - } - int *cell_id_popped; - int n_cell_id_popped; - int *cell_id_old; - int *cell_id_added; - int n_cell_id_added; - - cell_manager_merge_topk(cm1, cm2, &cell_id_popped, &n_cell_id_popped, &cell_id_old, &cell_id_added, &n_cell_id_added); - EXPECT_EQ(n_cell_id_popped, 0); - EXPECT_EQ(n_cell_id_added, 2); - EXPECT_EQ(cell_id_added[0], cell_id_1[0]); // key_share in cm1 - EXPECT_EQ(cell_id_old[0], cell_id_2[0]); // key_share in cm2 - EXPECT_EQ(cell_id_added[1], 3); // key_2, new cell - EXPECT_EQ(cell_id_old[1], cell_id_2[1]); // key_2 in cm2 - - auto test_result = test_query_cell_manager_content(cm1); - double accuracy = cal_accuracy_with_tags(keys, test_result); - EXPECT_NEAR(accuracy, 1.0, 0.01); - - EXPECT_EQ(cell_manager_get_count_by_tag(cm1, keys[0]), 2); // key_share merged once - - free(cell_id_popped); - free(cell_id_old); - free(cell_id_added); - cell_manager_free(cm1); - cell_manager_free(cm2); - - for (size_t i = 0; i < keys.size(); i++) { - tag_hash_key_free(keys[i]); - } -} - -TEST(unit_test_cell_manager, merge_topk_to_empty) -{ - struct cell_manager *cm1 = cell_manager_new(SAMPLING_MODE_TOPK, 10); - struct cell_manager *cm2 = cell_manager_new(SAMPLING_MODE_TOPK, 10); - - vector<struct tag_hash_key *> keys; - keys.push_back(test_gen_tag_key("key_1", 1)); - keys.push_back(test_gen_tag_key("key_1", 1)); - keys.push_back(test_gen_tag_key("key_1", 2)); - - int pop_dummy; - int cell_id_1[3]; - - for (size_t i = 0; i < 3; i++) { - cell_id_1[i] = cell_manager_add_cell_topk(cm1, keys[i], 1, &pop_dummy); - printf("cell id: %d\n", cell_id_1[i]); - } - - int *cell_id_popped; - int n_cell_id_popped; - int *cell_id_old; - int *cell_id_added; - int n_cell_id_added; - - cell_manager_merge_topk(cm2, cm1, &cell_id_popped, &n_cell_id_popped, &cell_id_old, &cell_id_added, &n_cell_id_added); - EXPECT_EQ(n_cell_id_popped, 0); - EXPECT_EQ(n_cell_id_added, 2); - EXPECT_EQ(cell_id_added[0], 0); // new added - EXPECT_EQ(cell_id_old[0], cell_id_1[0]); // key_11 in cm1 - EXPECT_EQ(cell_id_added[1], 1); // new added - EXPECT_EQ(cell_id_old[1], cell_id_1[2]); // key_12 in cm1 - EXPECT_EQ(cell_manager_get_count_by_tag(cm2, keys[0]), 2); // key_11 added twice - EXPECT_EQ(cell_manager_get_count_by_tag(cm2, keys[2]), 1); // key_12 - - free(cell_id_popped); - free(cell_id_old); - free(cell_id_added); - cell_manager_free(cm1); - cell_manager_free(cm2); - - for (size_t i = 0; i < keys.size(); i++) { - tag_hash_key_free(keys[i]); - } -} - -TEST(unit_test_cell_manager, merge_topk_to_full_one) -{ - struct cell_manager *cm1 = cell_manager_new(SAMPLING_MODE_TOPK, 10); - struct cell_manager *cm2 = cell_manager_new(SAMPLING_MODE_TOPK, 10); - - vector<struct tag_hash_key *> keys1; - keys1.push_back(test_gen_tag_key("key_1", 1)); - keys1.push_back(test_gen_tag_key("key_1", 2)); - keys1.push_back(test_gen_tag_key("key_shared", 1)); - - vector<struct tag_hash_key *> keys2; - for (int i = 0; i < 9; i++) { - keys2.push_back(test_gen_tag_key("key_2", i)); - } - keys2.push_back(test_gen_tag_key("key_shared", 1)); - - int pop_dummy; - int cell_id_1[3]; - int cell_id_2[10]; - for (size_t i = 0; i < 3; i++) { - cell_id_1[i] = cell_manager_add_cell_topk(cm1, keys1[i], 10, &pop_dummy); - } - for (size_t i = 0; i < 10; i++) { - unsigned int count = i < 2 ? i : 5; // the first 2 keys have count 1 and 2(less), the rest have count 5 - cell_id_2[i] = cell_manager_add_cell_topk(cm2, keys2[i], count, &pop_dummy); - } - - int *cell_id_popped; - int n_cell_id_popped; - int *cell_id_old; - int *cell_id_added; - int n_cell_id_added; - cell_manager_merge_topk(cm2, cm1, &cell_id_popped, &n_cell_id_popped, &cell_id_old, &cell_id_added, &n_cell_id_added); - - EXPECT_EQ(n_cell_id_popped, 2); // 2 "key_1" added, should pop 2 cells - EXPECT_EQ(cell_id_popped[0], cell_id_2[0]); // key2 which has count of 1 - EXPECT_EQ(cell_id_popped[1], cell_id_2[1]); // key2 which has count of 2 - EXPECT_EQ(n_cell_id_added, 3); // 2 "key_1", "key_shared" added, should add 3 cells - EXPECT_EQ(cell_id_added[0], 10); // newly added - EXPECT_EQ(cell_id_added[1], 11); // newly added - EXPECT_EQ(cell_id_added[2], cell_id_2[9]); // shared key - EXPECT_EQ(cell_id_old[0], cell_id_1[0]); // key_1 in cm1 - EXPECT_EQ(cell_id_old[1], cell_id_1[1]); // key_1 in cm1 - EXPECT_EQ(cell_id_old[2], cell_id_1[2]); // key_shared in cm1 - - auto test_result = test_query_cell_manager_content(cm2); - // join keys2 to keys1 - keys1.insert(keys1.end(), std::make_move_iterator(keys2.begin()), std::make_move_iterator(keys2.end())); - double accuracy = cal_accuracy_with_tags(keys1, test_result); - EXPECT_NEAR(accuracy, 1.0, 0.01); - - free(cell_id_popped); - free(cell_id_old); - free(cell_id_added); - cell_manager_free(cm1); - cell_manager_free(cm2); - for (size_t i = 0; i < keys1.size(); i++) { - tag_hash_key_free(keys1[i]); - } - // all keys are moved to cm1, so no need to free keys2 -} - -bool test_is_tag_equal(const struct tag_hash_key* a, const struct tag_hash_key* b) { - char *key1 = tag_hash_key_get_compound_key(a); - char *key2 = tag_hash_key_get_compound_key(b); - bool ret = strcmp(key1, key2) == 0; - free(key1); - free(key2); - return ret; -} - -void add_key_and_assert_find_result(struct cell_manager *cm, const struct tag_hash_key *key) -{ - int pop_dummy; - int cell_id = cell_manager_add_cell_topk(cm, key, 1234, &pop_dummy); - EXPECT_EQ(cell_id, 0); - EXPECT_EQ(cell_manager_get_count_by_tag(cm, key), 1234); - const struct tag_hash_key *key_get = cell_manager_get_tag_by_cell_id(cm, cell_id); - EXPECT_TRUE(test_is_tag_equal(key_get, key)); -} - -void add_key_and_assert_find_result_comprehensive(struct cell_manager *cm, const struct tag_hash_key *key) -{ - int cell_id = cell_manager_add_cell(cm, key); - EXPECT_EQ(cell_id, 0); - const struct tag_hash_key *key_get = cell_manager_get_tag_by_cell_id(cm, cell_id); - EXPECT_TRUE(test_is_tag_equal(key_get, key)); -} - -struct tag_hash_key *tag_hash_key_construct_with_fieldstat_tag(const struct fieldstat_tag *src, size_t n_src) -{ - struct tag_hash_key *ret_hash_key = (struct tag_hash_key *)malloc(sizeof(struct tag_hash_key)); - - tag_hash_key_init_with_fieldstat_tag(ret_hash_key, src, n_src, false); - - return ret_hash_key; -} - -TEST(unit_test_cell_manager, add_with_key_length_is_1_int_type_topk) -{ - struct cell_manager *cm = cell_manager_new(SAMPLING_MODE_TOPK, 10); - struct tag_hash_key *key = tag_hash_key_construct_with_fieldstat_tag(&TEST_TAG_INT, 1); - - add_key_and_assert_find_result(cm, key); - - cell_manager_free(cm); - tag_hash_key_free(key); -} - -TEST(unit_test_cell_manager, add_with_key_length_is_1_double_type_topk) -{ - struct tag_hash_key *key = tag_hash_key_construct_with_fieldstat_tag(&TEST_TAG_DOUBLE, 1); - struct cell_manager *cm = cell_manager_new(SAMPLING_MODE_TOPK, 10); - - add_key_and_assert_find_result(cm, key); - - cell_manager_free(cm); - tag_hash_key_free(key); -} - -TEST(unit_test_cell_manager, add_with_key_length_is_1_string_type_topk) -{ - struct tag_hash_key *key = tag_hash_key_construct_with_fieldstat_tag(&TEST_TAG_STRING, 1); - struct cell_manager *cm = cell_manager_new(SAMPLING_MODE_TOPK, 10); - - add_key_and_assert_find_result(cm, key); - - cell_manager_free(cm); - tag_hash_key_free(key); -} - -TEST(unit_test_cell_manager, add_with_key_length_is_3_of_diff_types_topk) -{ - const struct fieldstat_tag tags[3] = {TEST_TAG_INT, TEST_TAG_STRING, TEST_TAG_DOUBLE}; - struct tag_hash_key *key = tag_hash_key_construct_with_fieldstat_tag(tags, 3); - struct cell_manager *cm = cell_manager_new(SAMPLING_MODE_TOPK, 10); - - add_key_and_assert_find_result(cm, key); - - cell_manager_free(cm); - tag_hash_key_free(key); -} - -TEST(unit_test_cell_manager, add_with_key_length_is_1_int_type_comprehensive) -{ - struct cell_manager *cm = cell_manager_new(SAMPLING_MODE_COMPREHENSIVE, 10); - struct tag_hash_key *key = tag_hash_key_construct_with_fieldstat_tag(&TEST_TAG_INT, 1); - - add_key_and_assert_find_result_comprehensive(cm, key); - - cell_manager_free(cm); - tag_hash_key_free(key); -} - -TEST(unit_test_cell_manager, add_with_key_length_is_1_double_type_comprehensive) -{ - struct tag_hash_key *key = tag_hash_key_construct_with_fieldstat_tag(&TEST_TAG_DOUBLE, 1); - struct cell_manager *cm = cell_manager_new(SAMPLING_MODE_COMPREHENSIVE, 10); - - add_key_and_assert_find_result_comprehensive(cm, key); - - cell_manager_free(cm); - tag_hash_key_free(key); -} - -TEST(unit_test_cell_manager, add_with_key_length_is_1_string_type_comprehensive) -{ - struct tag_hash_key *key = tag_hash_key_construct_with_fieldstat_tag(&TEST_TAG_STRING, 1); - struct cell_manager *cm = cell_manager_new(SAMPLING_MODE_COMPREHENSIVE, 10); - - add_key_and_assert_find_result_comprehensive(cm, key); - - cell_manager_free(cm); - tag_hash_key_free(key); -} - -TEST(unit_test_cell_manager, add_with_key_length_is_3_of_diff_types_comprehensive) -{ - const struct fieldstat_tag tags[3] = {TEST_TAG_INT, TEST_TAG_STRING, TEST_TAG_DOUBLE}; - struct tag_hash_key *key = tag_hash_key_construct_with_fieldstat_tag(tags, 3); - struct cell_manager *cm = cell_manager_new(SAMPLING_MODE_COMPREHENSIVE, 10); - - add_key_and_assert_find_result_comprehensive(cm, key); - - cell_manager_free(cm); - tag_hash_key_free(key); -} - -extern "C" -{ - int find_next_unused_cell_id(const int *sorted_pst_cell_id_arr, size_t arr_len, int last_find_result, int *next_idx); -} -TEST(unit_test_cell_manager, find_next_id_given_all_cell_id_continuous) -{ - int *arr = (int *)malloc(sizeof(int) * 10); - for (int i = 0; i < 10; i++) - { - arr[i] = i; - } - - int last_find_result = -1; - int next_idx = 0; - last_find_result = find_next_unused_cell_id(arr, 10, last_find_result, &next_idx); - EXPECT_EQ(last_find_result, 10); - last_find_result = find_next_unused_cell_id(arr, 10, last_find_result, &next_idx); - EXPECT_EQ(last_find_result, 11); - - free(arr); -} - -TEST(unit_test_cell_manager, find_next_id_given_continuous_hole) -{ - int *arr = (int *)malloc(sizeof(int) * 10); - for (int i = 0; i < 3; i++) - { - arr[i] = i; - } - arr[3] = 5; // 3, 4 is hole - for (int i = 4; i < 10; i++) - { - arr[i] = i + 2; - } - - int last_find_result = -1; - int next_idx = 0; - last_find_result = find_next_unused_cell_id(arr, 10, last_find_result, &next_idx); - EXPECT_EQ(last_find_result, 3); - last_find_result = find_next_unused_cell_id(arr, 10, last_find_result, &next_idx); - EXPECT_EQ(last_find_result, 4); - last_find_result = find_next_unused_cell_id(arr, 10, last_find_result, &next_idx); - EXPECT_EQ(last_find_result, 12); - last_find_result = find_next_unused_cell_id(arr, 10, last_find_result, &next_idx); - EXPECT_EQ(last_find_result, 13); - - free(arr); -} - -TEST(unit_test_cell_manager, find_next_id_given_holes) -{ - int *arr = (int *)malloc(sizeof(int) * 10); - for (int i = 0; i < 10; i++) - { - arr[i] = i * 2; // 0, 2, 4, ... - } - - int last_find_result = -1; - int next_idx = 0; - last_find_result = find_next_unused_cell_id(arr, 10, last_find_result, &next_idx); - EXPECT_EQ(last_find_result, 1); - last_find_result = find_next_unused_cell_id(arr, 10, last_find_result, &next_idx); - EXPECT_EQ(last_find_result, 3); - last_find_result = find_next_unused_cell_id(arr, 10, last_find_result, &next_idx); - EXPECT_EQ(last_find_result, 5); - - free(arr); -} - -TEST(unit_test_cell_manager, find_next_id_given_holes_at_only_start) -{ - int *arr = (int *)malloc(sizeof(int ) * 10); - for (int i = 0; i < 10; i++) - { - arr[i] = i + 2; - } - - int last_find_result = -1; - int next_idx = 0; - last_find_result = find_next_unused_cell_id(arr, 10, last_find_result, &next_idx); - EXPECT_EQ(last_find_result, 0); - last_find_result = find_next_unused_cell_id(arr, 10, last_find_result, &next_idx); - EXPECT_EQ(last_find_result, 1); - last_find_result = find_next_unused_cell_id(arr, 10, last_find_result, &next_idx); - EXPECT_EQ(last_find_result, 12); - - free(arr); -} - -TEST(unit_test_cell_manager, find_next_id_given_empty_arr) -{ - int *arr = (int *)malloc(sizeof(int) * 0); - - int last_find_result = -1; - int next_idx = 0; - last_find_result = find_next_unused_cell_id(arr, 0, last_find_result, &next_idx); - EXPECT_EQ(last_find_result, 0); - last_find_result = find_next_unused_cell_id(arr, 0, last_find_result, &next_idx); - EXPECT_EQ(last_find_result, 1); - - free(arr); -} - -TEST(unit_test_cell_manager, reset_once_and_query) -{ - const int TEST_ROUND = 10; - struct cell_manager *cm = cell_manager_new(SAMPLING_MODE_COMPREHENSIVE, TEST_ROUND); - - vector<struct tag_hash_key *> keys; - for (int i = 0; i < TEST_ROUND; i++) - { - struct tag_hash_key *key = test_gen_tag_key("key", i); - keys.push_back(key); - } - for (int i = 0; i < TEST_ROUND; i++) { - int cell_id = cell_manager_add_cell(cm, keys[i]); - EXPECT_EQ(cell_id, i); - } - - cell_manager_reset(cm); - for (int i = 0; i < TEST_ROUND; i++) { - EXPECT_EQ(cell_manager_find(cm, keys[i]), -1); - EXPECT_TRUE(cell_manager_get_tag_by_cell_id(cm, i) == NULL); - } - int array_len = 0; - cell_manager_dump(cm, &array_len); - EXPECT_EQ(array_len, 0); - - // do it again, to see the effect of reset on a recovered cell_manager(instead of which just newed) - for (int i = 0; i < TEST_ROUND; i++) { - int cell_id = cell_manager_add_cell(cm, keys[i]); - EXPECT_EQ(cell_id, i); - } - cell_manager_reset(cm); - for (int i = 0; i < TEST_ROUND; i++) { - EXPECT_EQ(cell_manager_find(cm, keys[i]), -1); - EXPECT_TRUE(cell_manager_get_tag_by_cell_id(cm, i) == NULL); - } - cell_manager_dump(cm, &array_len); - EXPECT_EQ(array_len, 0); - - cell_manager_free(cm); - for (int i = 0; i < TEST_ROUND; i++) { - tag_hash_key_free(keys[i]); - } -} - -// reset once will not delete the cells, just let them be discared, so in such case, cube_add will just add to the same cell. -TEST(unit_test_cell_manager, add_after_reset_and_ensure_performance_improvement) { - clock_t start, end; - const int TEST_ROUND = 100000; - struct cell_manager *cm = cell_manager_new(SAMPLING_MODE_COMPREHENSIVE, TEST_ROUND); - vector<struct tag_hash_key *> keys; - for (int i = 0; i < TEST_ROUND; i++) - { - struct tag_hash_key *key = test_gen_tag_key("key", i); - keys.push_back(key); - } - for (int i = 0; i < TEST_ROUND; i++) { - int cell_id = cell_manager_add_cell(cm, keys[i]); - EXPECT_EQ(cell_id, i); - } - - cell_manager_reset(cm); - start = clock(); - for (int i = 0; i < TEST_ROUND; i++) - { - int cell_id = cell_manager_add_cell(cm, keys[i]); - EXPECT_EQ(cell_id, i); - } - end = clock(); - clock_t time_reset_once = end - start; - - - cell_manager_reset(cm); - cell_manager_reset(cm); - start = clock(); - for (int i = 0; i < TEST_ROUND; i++) - { - int cell_id = cell_manager_add_cell(cm, keys[i]); - EXPECT_EQ(cell_id, i); // completely new cell - } - end = clock(); - clock_t time_reset_twice = end - start; - - EXPECT_GT((double)time_reset_twice, (double)time_reset_once); - - cell_manager_free(cm); - for (int i = 0; i < TEST_ROUND; i++) { - tag_hash_key_free(keys[i]); - } -} - -TEST(unit_test_cell_manager, reset_once_and_query_topk) -{ - const int TEST_ROUND = 10; - struct cell_manager *cm = cell_manager_new(SAMPLING_MODE_TOPK, TEST_ROUND); - - vector<struct tag_hash_key *> keys; - for (int i = 0; i < TEST_ROUND; i++) - { - struct tag_hash_key *key = test_gen_tag_key("key", i); - keys.push_back(key); - } - for (int i = 0; i < TEST_ROUND; i++) { - int dummy; - int cell_id = cell_manager_add_cell_topk(cm, keys[i], 1, &dummy); - EXPECT_EQ(cell_id, i); - } - - cell_manager_reset(cm); - for (int i = 0; i < TEST_ROUND; i++) { - EXPECT_EQ(cell_manager_find(cm, keys[i]), -1); - EXPECT_TRUE(cell_manager_get_tag_by_cell_id(cm, i) == NULL); - EXPECT_EQ(cell_manager_get_count_by_tag(cm, keys[i]), -1); - } - int array_len = 0; - cell_manager_dump(cm, &array_len); - EXPECT_EQ(array_len, 0); - - cell_manager_free(cm); - for (int i = 0; i < TEST_ROUND; i++) { - tag_hash_key_free(keys[i]); - } -} - -// 速度测试 - -int main(int argc, char *argv[]) -{ - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -}
\ No newline at end of file diff --git a/test/utils.cpp b/test/utils.cpp index b53e4e4..4e41d8f 100644 --- a/test/utils.cpp +++ b/test/utils.cpp @@ -8,7 +8,10 @@ #include <random> #include <string.h> #include <algorithm> - +#include <fstream> +#include <math.h> +#include <unistd.h> +#include <sstream> #include "fieldstat.h" #include "utils.hpp" @@ -32,24 +35,24 @@ string gen_rand_string(int len) /* taglist wrapper */ /* -------------------------------------------------------------------------- */ -Fieldstat_tag_list_wrapper::Fieldstat_tag_list_wrapper(const struct fieldstat_tag_list *tag_list) { - tag_list_c.tag = (struct fieldstat_tag *)malloc(sizeof(struct fieldstat_tag) * tag_list->n_tag); - tag_list_c.n_tag = tag_list->n_tag; - for (size_t i = 0; i < tag_list->n_tag; i++) +Fieldstat_tag_list_wrapper::Fieldstat_tag_list_wrapper(const struct field_list *tag_list) { + tag_list_c.field = (struct field *)malloc(sizeof(struct field) * tag_list->n_field); + tag_list_c.n_field = tag_list->n_field; + for (size_t i = 0; i < tag_list->n_field; i++) { // copy the tag_list - tag_list_c.tag[i].key = strdup(tag_list->tag[i].key); - tag_list_c.tag[i].type = tag_list->tag[i].type; - switch (tag_list->tag[i].type) + tag_list_c.field[i].key = strdup(tag_list->field[i].key); + tag_list_c.field[i].type = tag_list->field[i].type; + switch (tag_list->field[i].type) { - case TAG_INTEGER: - tag_list_c.tag[i].value_longlong = tag_list->tag[i].value_longlong; + case FIELD_VALUE_INTEGER: + tag_list_c.field[i].value_longlong = tag_list->field[i].value_longlong; break; - case TAG_DOUBLE: - tag_list_c.tag[i].value_double = tag_list->tag[i].value_double; + case FIELD_VALUE_DOUBLE: + tag_list_c.field[i].value_double = tag_list->field[i].value_double; break; - case TAG_CSTRING: - tag_list_c.tag[i].value_str = strdup(tag_list->tag[i].value_str); + case FIELD_VALUE_CSTRING: + tag_list_c.field[i].value_str = strdup(tag_list->field[i].value_str); break; default: break; @@ -59,84 +62,84 @@ Fieldstat_tag_list_wrapper::Fieldstat_tag_list_wrapper(const struct fieldstat_ta Fieldstat_tag_list_wrapper::Fieldstat_tag_list_wrapper(const char * key, int value) { - tag_list_c.tag = (struct fieldstat_tag *)malloc(sizeof(struct fieldstat_tag)); - tag_list_c.n_tag = 1; - tag_list_c.tag[0].key = strdup(key); - tag_list_c.tag[0].type = TAG_INTEGER; - tag_list_c.tag[0].value_longlong = value; + tag_list_c.field = (struct field *)malloc(sizeof(struct field)); + tag_list_c.n_field = 1; + tag_list_c.field[0].key = strdup(key); + tag_list_c.field[0].type = FIELD_VALUE_INTEGER; + tag_list_c.field[0].value_longlong = value; } Fieldstat_tag_list_wrapper::Fieldstat_tag_list_wrapper(const char * key, const char *value) { - tag_list_c.tag = (struct fieldstat_tag *)malloc(sizeof(struct fieldstat_tag)); - tag_list_c.n_tag = 1; - tag_list_c.tag[0].key = strdup(key); - tag_list_c.tag[0].type = TAG_CSTRING; - tag_list_c.tag[0].value_str = strdup(value); + tag_list_c.field = (struct field *)malloc(sizeof(struct field)); + tag_list_c.n_field = 1; + tag_list_c.field[0].key = strdup(key); + tag_list_c.field[0].type = FIELD_VALUE_CSTRING; + tag_list_c.field[0].value_str = strdup(value); } Fieldstat_tag_list_wrapper::~Fieldstat_tag_list_wrapper() { - for (size_t i = 0; i < tag_list_c.n_tag; i++) { - free((char *)tag_list_c.tag[i].key); - if (tag_list_c.tag[i].type == TAG_CSTRING) { - free((char *)tag_list_c.tag[i].value_str); + for (size_t i = 0; i < tag_list_c.n_field; i++) { + free((char *)tag_list_c.field[i].key); + if (tag_list_c.field[i].type == FIELD_VALUE_CSTRING) { + free((char *)tag_list_c.field[i].value_str); } } - free(tag_list_c.tag); + free(tag_list_c.field); } Fieldstat_tag_list_wrapper::Fieldstat_tag_list_wrapper(std::uniform_int_distribution<int> &dist, int tag_count) { - tag_list_c.tag = (struct fieldstat_tag *)malloc(sizeof(struct fieldstat_tag) * tag_count); - tag_list_c.n_tag = tag_count; + tag_list_c.field = (struct field *)malloc(sizeof(struct field) * tag_count); + tag_list_c.n_field = tag_count; std::mt19937 rng(1); for (int i = 0; i < tag_count; i++) { - tag_list_c.tag[i].key = strdup(gen_rand_string(10).c_str()); + tag_list_c.field[i].key = strdup(gen_rand_string(10).c_str()); int rand_ret = rand() % 3; if (rand_ret == 0) { - tag_list_c.tag[i].type = TAG_INTEGER; - tag_list_c.tag[i].value_longlong = static_cast<long long>(dist(rng)); + tag_list_c.field[i].type = FIELD_VALUE_INTEGER; + tag_list_c.field[i].value_longlong = static_cast<long long>(dist(rng)); } else if (rand_ret == 1) { - tag_list_c.tag[i].type = TAG_DOUBLE; - tag_list_c.tag[i].value_double = static_cast<double>(dist(rng)) + 0.5; + tag_list_c.field[i].type = FIELD_VALUE_DOUBLE; + tag_list_c.field[i].value_double = static_cast<double>(dist(rng)) + 0.5; } else { - tag_list_c.tag[i].type = TAG_CSTRING; - tag_list_c.tag[i].value_str = strdup(gen_rand_string(10).c_str()); + tag_list_c.field[i].type = FIELD_VALUE_CSTRING; + tag_list_c.field[i].value_str = strdup(gen_rand_string(10).c_str()); } } } Fieldstat_tag_list_wrapper::Fieldstat_tag_list_wrapper() { - tag_list_c.tag = NULL; - tag_list_c.n_tag = 0; + tag_list_c.field = NULL; + tag_list_c.n_field = 0; } Fieldstat_tag_list_wrapper::Fieldstat_tag_list_wrapper(const Fieldstat_tag_list_wrapper &tag_list_wrapper){ - const struct fieldstat_tag_list *tag_list = tag_list_wrapper.get_c_struct(); - tag_list_c.tag = (struct fieldstat_tag *)malloc(sizeof(struct fieldstat_tag) * tag_list->n_tag); - tag_list_c.n_tag = tag_list->n_tag; - for (size_t i = 0; i < tag_list->n_tag; i++) + const struct field_list *tag_list = tag_list_wrapper.get_c_struct(); + tag_list_c.field = (struct field *)malloc(sizeof(struct field) * tag_list->n_field); + tag_list_c.n_field = tag_list->n_field; + for (size_t i = 0; i < tag_list->n_field; i++) { // copy the tag_list - tag_list_c.tag[i].key = strdup(tag_list->tag[i].key); - tag_list_c.tag[i].type = tag_list->tag[i].type; - switch (tag_list->tag[i].type) + tag_list_c.field[i].key = strdup(tag_list->field[i].key); + tag_list_c.field[i].type = tag_list->field[i].type; + switch (tag_list->field[i].type) { - case TAG_INTEGER: - tag_list_c.tag[i].value_longlong = tag_list->tag[i].value_longlong; + case FIELD_VALUE_INTEGER: + tag_list_c.field[i].value_longlong = tag_list->field[i].value_longlong; break; - case TAG_DOUBLE: - tag_list_c.tag[i].value_double = tag_list->tag[i].value_double; + case FIELD_VALUE_DOUBLE: + tag_list_c.field[i].value_double = tag_list->field[i].value_double; break; - case TAG_CSTRING: - tag_list_c.tag[i].value_str = strdup(tag_list->tag[i].value_str); + case FIELD_VALUE_CSTRING: + tag_list_c.field[i].value_str = strdup(tag_list->field[i].value_str); break; default: break; @@ -144,38 +147,38 @@ Fieldstat_tag_list_wrapper::Fieldstat_tag_list_wrapper(const Fieldstat_tag_list_ } } -const struct fieldstat_tag *Fieldstat_tag_list_wrapper::get_tag() const +const struct field *Fieldstat_tag_list_wrapper::get_tag() const { - return tag_list_c.tag; + return tag_list_c.field; } size_t Fieldstat_tag_list_wrapper::get_tag_count() const { - return tag_list_c.n_tag; + return tag_list_c.n_field; } -const struct fieldstat_tag_list *Fieldstat_tag_list_wrapper::get_c_struct() const +const struct field_list *Fieldstat_tag_list_wrapper::get_c_struct() const { return &tag_list_c; } void Fieldstat_tag_list_wrapper::print_tag_list() const { - printf("tag_list_c.n_tag: %zu\n", tag_list_c.n_tag); - for (size_t i = 0; i < tag_list_c.n_tag; i++) + printf("tag_list_c.n_field: %zu\n", tag_list_c.n_field); + for (size_t i = 0; i < tag_list_c.n_field; i++) { - printf("tag_list_c.tag[%zu].key: %s\n", i, tag_list_c.tag[i].key); - printf("tag_list_c.tag[%zu].type: %d\n", i, (int)tag_list_c.tag[i].type); - switch (tag_list_c.tag[i].type) + printf("tag_list_c.field[%zu].key: %s\n", i, tag_list_c.field[i].key); + printf("tag_list_c.field[%zu].type: %d\n", i, (int)tag_list_c.field[i].type); + switch (tag_list_c.field[i].type) { - case TAG_INTEGER: - printf("tag_list_c.tag[%zu].value_longlong: %lld\n", i, tag_list_c.tag[i].value_longlong); + case FIELD_VALUE_INTEGER: + printf("tag_list_c.field[%zu].value_longlong: %lld\n", i, tag_list_c.field[i].value_longlong); break; - case TAG_DOUBLE: - printf("tag_list_c.tag[%zu].value_double: %lf\n", i, tag_list_c.tag[i].value_double); + case FIELD_VALUE_DOUBLE: + printf("tag_list_c.field[%zu].value_double: %lf\n", i, tag_list_c.field[i].value_double); break; - case TAG_CSTRING: - printf("tag_list_c.tag[%zu].value_str: %s\n", i, tag_list_c.tag[i].value_str); + case FIELD_VALUE_CSTRING: + printf("tag_list_c.field[%zu].value_str: %s\n", i, tag_list_c.field[i].value_str); break; default: break; @@ -187,20 +190,20 @@ void Fieldstat_tag_list_wrapper::print_tag_list() const string Fieldstat_tag_list_wrapper::to_string() const { string str = ""; - for (size_t i = 0; i < tag_list_c.n_tag; i++) + for (size_t i = 0; i < tag_list_c.n_field; i++) { - str += tag_list_c.tag[i].key; + str += tag_list_c.field[i].key; str += ":"; - switch (tag_list_c.tag[i].type) + switch (tag_list_c.field[i].type) { - case TAG_INTEGER: - str += std::to_string(tag_list_c.tag[i].value_longlong); + case FIELD_VALUE_INTEGER: + str += std::to_string(tag_list_c.field[i].value_longlong); break; - case TAG_DOUBLE: - str += std::to_string(tag_list_c.tag[i].value_double); + case FIELD_VALUE_DOUBLE: + str += std::to_string(tag_list_c.field[i].value_double); break; - case TAG_CSTRING: - str += tag_list_c.tag[i].value_str; + case FIELD_VALUE_CSTRING: + str += tag_list_c.field[i].value_str; break; default: break; @@ -212,30 +215,30 @@ string Fieldstat_tag_list_wrapper::to_string() const bool Fieldstat_tag_list_wrapper::operator==(const Fieldstat_tag_list_wrapper &tag_list_wrapper) const { - const struct fieldstat_tag_list *tag_list = tag_list_wrapper.get_c_struct(); - if (tag_list_c.n_tag != tag_list->n_tag) { + const struct field_list *tag_list = tag_list_wrapper.get_c_struct(); + if (tag_list_c.n_field != tag_list->n_field) { return false; } - for (size_t i = 0; i < tag_list_c.n_tag; i++) { - if (strcmp((char *)tag_list_c.tag[i].key, (char *)tag_list->tag[i].key) != 0) { + for (size_t i = 0; i < tag_list_c.n_field; i++) { + if (strcmp((char *)tag_list_c.field[i].key, (char *)tag_list->field[i].key) != 0) { return false; } - if (tag_list_c.tag[i].type != tag_list->tag[i].type) { + if (tag_list_c.field[i].type != tag_list->field[i].type) { return false; } - switch (tag_list_c.tag[i].type) { - case TAG_INTEGER: - if (tag_list_c.tag[i].value_longlong != tag_list->tag[i].value_longlong) { + switch (tag_list_c.field[i].type) { + case FIELD_VALUE_INTEGER: + if (tag_list_c.field[i].value_longlong != tag_list->field[i].value_longlong) { return false; } break; - case TAG_DOUBLE: - if (tag_list_c.tag[i].value_double != tag_list->tag[i].value_double) { + case FIELD_VALUE_DOUBLE: + if (tag_list_c.field[i].value_double != tag_list->field[i].value_double) { return false; } break; - case TAG_CSTRING: - if (strcmp((char *)tag_list_c.tag[i].value_str, (char *)tag_list->tag[i].value_str) != 0) { + case FIELD_VALUE_CSTRING: + if (strcmp((char *)tag_list_c.field[i].value_str, (char *)tag_list->field[i].value_str) != 0) { return false; } break; @@ -248,7 +251,7 @@ bool Fieldstat_tag_list_wrapper::operator==(const Fieldstat_tag_list_wrapper &ta Fieldstat_tag_list_wrapper& Fieldstat_tag_list_wrapper::sort_tag_list() { - std::sort(tag_list_c.tag, tag_list_c.tag + tag_list_c.n_tag, [](const struct fieldstat_tag &a, const struct fieldstat_tag &b) { + std::sort(tag_list_c.field, tag_list_c.field + tag_list_c.n_field, [](const struct field &a, const struct field &b) { return strcmp((char *)a.key, (char *)b.key) < 0; }); return *this; @@ -293,3 +296,115 @@ double test_cal_topk_accuracy(vector<struct Fieldstat_tag_list_wrapper *> &test_ double accuracy = (double)correct / test_result.size(); return accuracy; } + + +//=========================================================================== +//= Function to generate Zipf (power law) distributed random variables = +//= - Input: alpha and N = +//= - Output: Returns with Zipf distributed random variable = +//=========================================================================== +int zipf(double alpha, int n) +{ + static bool first = true; // Static first time flag + static double c = 0; // Normalization constant + double z; // Uniform random number (0 < z < 1) + double sum_prob; // Sum of probabilities + double zipf_value; // Computed exponential value to be returned + int i; // Loop counter + + // Compute normalization constant on first call only + if (first) + { + for (i=1; i<=n; i++) + c = c + (1.0 / pow((double) i, alpha)); + c = 1.0 / c; + first = false; + } + + // Pull a uniform random number (0 < z < 1) + do + { + z = (double)rand() / (double)RAND_MAX; + } + while ((z == 0.0) || (z == 1.0)); + + // Map z to the value + sum_prob = 0; + for (i=1; i<=n; i++) + { + sum_prob = sum_prob + c / pow((double) i, alpha); + if (sum_prob >= z) + { + zipf_value = i; + break; + } + } + + return(zipf_value); +} + +SpreadSketchZipfGenerator::SpreadSketchZipfGenerator(double alpha, int n) { + _alpha = alpha; + _n = n; + cursor = 0; + + // generate data and write them to file + std::string filename = "zipf_" + std::to_string(alpha) + "_" + std::to_string(n) + ".txt"; + + std::unordered_map<int, int> fanout_map; // src_ip_id -> fanout being used + + if (access(filename.c_str(), F_OK) != 0) { + printf("file %s not found, generating data\n", filename.c_str()); + + std::ofstream file(filename); + if (!file.is_open()) { + printf("failed to open file %s\n", filename.c_str()); + return; + } + + for (int i = 0; i < MAX_DATA; i++) { + int src_id = zipf(alpha, n); + int fanout = fanout_map.find(src_id) == fanout_map.end() ? 0 : fanout_map[src_id]; + fanout_map[src_id] = fanout + 1; + + file << "s_" << src_id << " d_" << fanout << std::endl; + } + + file.close(); + printf("data generated and saved to file %s\n", filename.c_str()); + } + + // load data + std::ifstream file(filename); + if (!file.is_open()) { + printf("failed to open file %s\n", filename.c_str()); + return; + } + + loadeds = new std::vector<std::pair<std::string, std::string>>; + std::string line; + while (std::getline(file, line)) { + std::istringstream iss(line); + std::string src_ip, dst_ip; + iss >> src_ip >> dst_ip; + loadeds->push_back(std::make_pair(src_ip, dst_ip)); + } + file.close(); + +} + +SpreadSketchZipfGenerator::~SpreadSketchZipfGenerator() { + delete loadeds; +} + +struct Flow SpreadSketchZipfGenerator::next() { + int r_cursor = cursor % loadeds->size(); + struct Flow flow; + + flow.src_ip = loadeds->at(r_cursor).first; + flow.dst_ip = loadeds->at(r_cursor).second; + + cursor++; + + return flow; +}
\ No newline at end of file diff --git a/test/utils.hpp b/test/utils.hpp index 4fe7fde..efac03a 100644 --- a/test/utils.hpp +++ b/test/utils.hpp @@ -3,15 +3,14 @@ #include <unordered_map> #include "fieldstat.h" -const struct fieldstat_tag TEST_TAG_STRING = {"STRING KEY_", TAG_CSTRING, {.value_str = "100.1"}}; -const struct fieldstat_tag_list TEST_TAG_LIST_STRING = {(struct fieldstat_tag *)&TEST_TAG_STRING, 1}; -const struct fieldstat_tag TEST_TAG_STRING_collided = {"collided", TAG_CSTRING, {.value_str = "2"}}; -const struct fieldstat_tag TEST_TAG_INT = {"INT key_", TAG_INTEGER, {.value_longlong = 100}}; -const struct fieldstat_tag_list TEST_TAG_LIST_INT = {(struct fieldstat_tag *)&TEST_TAG_INT, 1}; -const struct fieldstat_tag TEST_TAG_INT_collided = {"collided", TAG_INTEGER, {.value_longlong = 2}}; -const struct fieldstat_tag TEST_TAG_DOUBLE = {"DOUBLE key_", TAG_DOUBLE, {.value_double = 100.1}}; -const struct fieldstat_tag TEST_TAG_DOUBLE_collided = {"collided", TAG_DOUBLE, {.value_double = 2.0}}; -const struct fieldstat_tag TEST_SHARED_TAG = {"shared", TAG_INTEGER, {.value_longlong = 1}}; +const struct field TEST_FIELD_STRING = {"STRING KEY_", FIELD_VALUE_CSTRING, {.value_str = "100.1"}}; +const struct field_list TEST_FIELD_LIST_STRING = {(struct field *)&TEST_FIELD_STRING, 1}; +const struct field TEST_FIELD_INT = {"INT key_", FIELD_VALUE_INTEGER, {.value_longlong = 100}}; +const struct field_list TEST_FIELD_LIST_INT = {(struct field *)&TEST_FIELD_INT, 1}; +const struct field TEST_FIELD_INT_2 = {"collided", FIELD_VALUE_INTEGER, {.value_longlong = 2}}; +const struct field TEST_FIELD_DOUBLE = {"DOUBLE key_", FIELD_VALUE_DOUBLE, {.value_double = 100.1}}; +const struct field_list TEST_FIELD_LIST_DOUBLE = {(struct field *)&TEST_FIELD_DOUBLE, 1}; +const struct field TEST_SHARED_TAG = {"shared", FIELD_VALUE_INTEGER, {.value_longlong = 1}}; const struct timeval TEST_TIMEVAL = {100, 10000}; const long long TEST_TIMEVAL_LONG = 100010; // 100s * 1000 + 10000us / 1000 = 100010ms @@ -19,7 +18,7 @@ std::string gen_rand_string(int len); class Fieldstat_tag_list_wrapper { public: - explicit Fieldstat_tag_list_wrapper(const struct fieldstat_tag_list *tag_list_c); + explicit Fieldstat_tag_list_wrapper(const struct field_list *tag_list_c); explicit Fieldstat_tag_list_wrapper(const char * key, int value); explicit Fieldstat_tag_list_wrapper(const char * key, const char *value); explicit Fieldstat_tag_list_wrapper(std::uniform_int_distribution<int> &dist, int tag_count); @@ -30,17 +29,38 @@ public: ~Fieldstat_tag_list_wrapper(); bool operator==(const Fieldstat_tag_list_wrapper &tag_list_wrapper) const; - const struct fieldstat_tag *get_tag() const; + const struct field *get_tag() const; size_t get_tag_count() const; - const struct fieldstat_tag_list *get_c_struct() const; + const struct field_list *get_c_struct() const; void print_tag_list() const; Fieldstat_tag_list_wrapper& sort_tag_list(); private: - struct fieldstat_tag_list tag_list_c; + struct field_list tag_list_c; }; double test_cal_topk_accuracy(std::vector<struct Fieldstat_tag_list_wrapper *> &test_result, std::unordered_map<std::string, int> &expected_count); // after we change fieldstat_counter_get return a error code in, all the tests should change correspondingly, so just use a adapter aliasing the old function -long long my_fieldstat_counter_get(const struct fieldstat *instance, int cube_id, int metric_id, int cell_id);
\ No newline at end of file +long long my_fieldstat_counter_get(const struct fieldstat *instance, int cube_id, int metric_id, int cell_id); + + +struct Flow { + std::string src_ip; + std::string dst_ip; +}; + +class SpreadSketchZipfGenerator { +private: + const int MAX_DATA = 1000000; + std::vector<std::pair<std::string, std::string>> *loadeds; + unsigned cursor; + +public: + SpreadSketchZipfGenerator(double alpha, int n); + struct Flow next(); + ~SpreadSketchZipfGenerator(); + + double _alpha; + int _n; +};
\ No newline at end of file diff --git a/vendors/mpack/mpack.h b/vendors/mpack/mpack.h index 1f2386a..26694f5 100644 --- a/vendors/mpack/mpack.h +++ b/vendors/mpack/mpack.h @@ -2387,7 +2387,7 @@ MPACK_INLINE uint32_t mpack_tag_array_count(mpack_tag_t* tag) { * * @see mpack_type_map */ -MPACK_INLINE uint32_t mpack_tag_map_count(mpack_tag_t* tag) { +MPACK_INLINE uint32_t mpack_hash_table_count(mpack_tag_t* tag) { mpack_assert(tag->type == mpack_type_map, "tag is not a map!"); return tag->v.n; } |