#include #include #include #include #include "fieldstat.h" #include "fieldstat_internal.h" #include "cJSON.h" #include #include /* date: 2023/9/5 spinlock, output interval: 1ms, Test case: RecordExecTime -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Thread num 1 | Thread num 2 | Thread num 4 | Thread num 8 | Thread num 16 | Thread num 32 | Thread num 64 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | operate counter | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1 | 45 | 17 | 18 | 28 | 48 | 41 | 35 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10 | 21 | 19 | 19 | 17 | 13 | 10 | 9 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100 | 49 | 33 | 33 | 34 | 32 | 32 | 30 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1,000 | 217 | 284 | 321 | 719 | 536 | 803 | 1202 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10,000 | 1,961 | 2,618 | 3,742 | 7,068 | 9,943 | 19,171 | 16,970 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100,000 | 19,413 | 29,401 | 35,992 | 65,337 | 97,261 | 135,835 | 197,634 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1,000,000 | 193,769 | 278,782 | 359,098 | 883,753 | 1,493,073 | 1,592,048 | 1,974,276 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10,000,000 | 1,948,891 | 3,861,912 | 5,796,965 | 7,507,053 | 11,502,201 | 17,721,148 | 20,532,196 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100,000,000 | 19,425,541 | 30,764,254 | 57,262,235 | 97,336,038 | 117,999,613 | 156,962,386 | 203,782,206 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- no spinlock, output interval: 1ms, Test case: RecordExecTime -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Thread num 1 | Thread num 2 | Thread num 4 | Thread num 8 | Thread num 16 | Thread num 32 | Thread num 64 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | operate counter | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1 | 39 | 22 | 25 | 81 | 65 | 41 | 45 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10 | 34 | 12 | 13 | 8 | 11 | 11 | 9 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100 | 30 | 32 | 30 | 29 | 30 | 30 | 27 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1,000 | 201 | 205 | 201 | 205 | 207 | 204 | 202 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10,000 | 1,928 | 1,912 | 1,901 | 1,905 | 1,901 | 19,215 | 1,967 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100,000 | 18,742 | 18,674 | 18,917 | 18,801 | 18,815 | 193,246 | 23,323 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1,000,000 | 187,415 | 189,673 | 188,229 | 188,012 | 187,757 | 193,246 | 309,948 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10,000,000 | 1,879,720 | 1,888,791 | 1,880,554 | 1,883,515 | 1,882,172 | 1,892,792 | 2,681,721 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100,000,000 | 18,784,181 | 18,827,600 | 18,812,896 | 18,875,007 | 18,833,475 | 18,811,591 | 26,965,032 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- spinlock with Struct Alignment, output interval: 1ms, Test case: RecordExecTime -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Thread num 1 | Thread num 2 | Thread num 4 | Thread num 8 | Thread num 16 | Thread num 32 | Thread num 64 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | operate counter | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1 | 46 | 18 | 19 | 22 | 40 | 53 | 38 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10 | 25 | 13 | 14 | 12 | 12 | 10 | 9 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100 | 33 | 31 | 33 | 30 | 32 | 30 | 30 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1,000 | 212 | 207 | 536 | 205 | 207 | 207 | 204 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10,000 | 1,882 | 1,895 | 4,333 | 1,890 | 1,899 | 1,898 | 1,907 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100,000 | 18,751 | 18,823 | 32,792 | 19,100 | 18,839 | 18,860 | 20,919 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1,000,000 | 187,274 | 187,520 | 187,618 | 188,000 | 188,097 | 191,853 | 248,770 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10,000,000 | 18,731,69 | 1,874,701 | 1,875,510 | 1,875,648 | 1,878,946 | 1,919,397 | 2,523,767 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100,000,000 | 18,776,370 | 18,781,233 | 18,767,859 | 18,768,590 | 18,830,596 | 18,799,717 | 27,003,506 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- */ /* date: 2023/9/6 no lock, output interval 1ms, Test case: RecordExecTime -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Thread num 1 | Thread num 2 | Thread num 4 | Thread num 8 | Thread num 16 | Thread num 32 | Thread num 64 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | operate counter | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1 | 42 | 28 | 17 | 22 | 59 | 42 | 40 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10 | 29 | 14 | 10 | 11 | 12 | 11 | 9 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100 | 32 | 31 | 34 | 31 | 32 | 30 | 29 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1,000 | 200 | 208 | 208 | 212 | 213 | 212 | 208 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10,000 | 1,989 | 1,975 | 1,953 | 1,932 | 1,952 | 1,958 | 1,948 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100,000 | 20,193 | 19,209 | 19,306 | 19,126 | 19,257 | 19,336 | 21,010 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1,000,000 | 193,482 | 191,674 | 192,383 | 192,317 | 193,822 | 192,023 | 254,656 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10,000,000 | 1,936,361 | 1,931,578 | 1,936,618 | 1,921,114 | 1,926,897 | 1,920,143 | 2,595,562 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100,000,000 | 19,245,473 | 19,243,094 | 19,204,994 | 19,193,001 | 19,196,800 | 19,181,999 | 27,619,589 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- spinlock, output interval 1ms, Test case: RecordExecTime -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Thread num 1 | Thread num 2 | Thread num 4 | Thread num 8 | Thread num 16 | Thread num 32 | Thread num 64 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | operate counter | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1 | 59 | 18 | 20 | 24 | 38 | 40 | 38 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10 | 27 | 13 | 12 | 13 | 11 | 11 | 9 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100 | 29 | 39 | 32 | 30 | 31 | 30 | 29 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1,000 | 209 | 211 | 327 | 203 | 207 | 206 | 203 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10,000 | 1,896 | 1,897 | 3,162 | 1,893 | 1,897 | 1,905 | 1,908 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100,000 | 18,734 | 18,718 | 31,251 | 18,751 | 18,738 | 18,820 | 20,910 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1,000,000 | 186,970 | 187,683 | 187,262 | 187,649 | 188,037 | 188,369 | 247,677 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10,000,000 | 1,877,762 | 1,880,236 | 1,876,492 | 1,875,297 | 1,878,232 | 1,879,564 | 2,603,063 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100,000,000 | 18,720,506 | 18,736,390 | 18,740,244 | 18,769,749 | 18,779,667 | 18,806,123 | 26,277,081 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- rwlock, output interval 1ms, Test case: RecordExecTime -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Thread num 1 | Thread num 2 | Thread num 4 | Thread num 8 | Thread num 16 | Thread num 32 | Thread num 64 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | operate counter | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1 | 52 | 18 | 18 | 48 | 61 | 41 | 44 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10 | 30 | 13 | 13 | 11 | 12 | 10 | 19 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100 | 42 | 37 | 34 | 34 | 34 | 31 | 32 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1,000 | 230 | 222 | 377 | 311 | 224 | 222 | 345 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10,000 | 2,101 | 2,369 | 6,710 | 2,944 | 3,537 | 2,069 | 3,560 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100,000 | 20,448 | 24,518 | 39,459 | 30,087 | 34,189 | 20,981 | 38,270 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1,000,000 | 204,923 | 272,109 | 205,070 | 307,572 | 397,855 | 209,094 | 411,857 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10,000,000 | 2,043,604 | 2,048,087 | 3,021,970 | 2,933,224 | 2,070,708 | 2,085,169 | 4,453,767 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100,000,000 | 20,413,678 | 20,465,977 | 26,326,140 | 29,201,649 | 20,692,983 | 20,822,775 | 46,712,568 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- mutex, output interval 1ms, Test case: RecordExecTime -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Thread num 1 | Thread num 2 | Thread num 4 | Thread num 8 | Thread num 16 | Thread num 32 | Thread num 64 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | operate counter | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1 | 53 | 15 | 29 | 24 | 41 | 44 | 40 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10 | 34 | 13 | 13 | 13 | 13 | 12 | 10 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100 | 37 | 34 | 33 | 31 | 35 | 31 | 31 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1,000 | 220 | 224 | 614 | 223 | 225 | 223 | 224 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10,000 | 2,108 | 2,113 | 4,294 | 2,111 | 2,110 | 2,132 | 2,167 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100,000 | 21,089 | 20,854 | 65,000 | 21,106 | 21,021 | 21,553 | 24,513 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1,000,000 | 210,556 | 270,618 | 209,851 | 209,477 | 382,730 | 215,677 | 285,425 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10,000,000 | 2,091,623 | 2,095,177 | 2,096,735 | 2,093,514 | 2,114,204 | 2,203,298 | 2,920,734 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100,000,000 | 20,968,000 | 20,874,788 | 20,943,953 | 20,944,974 | 21,137,730 | 21,657,745 | 30,985,793 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- */ /* date: 2023/9/20 no lock, output interval 1ms, Test case: RecordExecTime -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Thread num 1 | Thread num 2 | Thread num 4 | Thread num 8 | Thread num 16 | Thread num 32 | Thread num 64 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | operate counter | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1 | 28 | 11 | 11 | 13 | 28 | 28 | 25 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10 | 11 | 8 | 8 | 8 | 9 | 7 | 6 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100 | 31 | 28 | 30 | 25 | 28 | 25 | 24 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1,000 | 198 | 200 | 198 | 198 | 198 | 198 | 198 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10,000 | 1,878 | 1,889 | 1,882 | 1,879 | 1,886 | 1,884 | 1,882 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100,000 | 18,724 | 18,627 | 18,636 | 18,678 | 18,695 | 19,995 | 19,479 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 1,000,000 | 187,366 | 186,860 | 186,783 | 186,636 | 186,446 | 188,050 | 239,472 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 10,000,000 | 1,865,582 | 1,863,954 | 1,862,493 | 1,863,763 | 1,867,006 | 1,869,409 | 2,450,149 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 100,000,000 | 18,635,991 | 18,656,564 | 18,636,838 | 18,644,694 | 18,678,975 | 18,656,275 | 25,055,446 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- */ struct thread_para { int loops; int thread_id; long long duration; struct fieldstat_dynamic_instance *instance; }; long long current_timestamp() { struct timeval te; gettimeofday(&te, NULL); return te.tv_sec * 1000LL * 1000LL + te.tv_usec; } void _worker_thread_one_metric(void *arg) { struct thread_para *para = (struct thread_para*)arg; int loops = para->loops; int thread_id = para->thread_id; struct fieldstat_dynamic_instance *instance = para->instance; //char metric_name[128] = {0}; int ret = 0; long long start_time, end_time; start_time = current_timestamp(); for(int i = 0; i < loops; i++) { //memset(metric_name, 0, sizeof(metric_name)); //snprintf(metric_name, sizeof(metric_name), "Active_sessions_%d", i); ret = fieldstat_dynamic_metric_value_incrby(instance, FIELD_METRIC_TYPE_GAUGE, "Active_sessions", 10, NULL, 0, thread_id); EXPECT_EQ(0, ret); } end_time =current_timestamp(); para->duration = end_time - start_time; return; } void *worker_thread_one_metric(void *arg) { _worker_thread_one_metric(arg); usleep(1000 * 100); return NULL; } static void fieldstat_dynamic_benchmark(int n_thread, int n_loops) { int ret = 0; //int n_thread = 4; //int n_loops = 10; struct fieldstat_dynamic_instance *instance = NULL; struct thread_para para[n_thread]; pthread_t thread_ids[n_thread]; long long sum_durations = 0; instance = fieldstat_dynamic_instance_new("firewall", n_thread); ret = fieldstat_dynamic_set_line_protocol_server(instance, "127.0.0.1", 8700); EXPECT_EQ(0, ret); // ret = fieldstat_dynamic_set_output_interval(instance, 1); // EXPECT_EQ(0, ret); fieldstat_dynamic_instance_start(instance); for(int i = 0; i < n_thread; i++) { para[i].loops = n_loops; para[i].instance = instance; para[i].thread_id = i; } for(int i = 0; i < n_thread; i++) { ret = pthread_create(&(thread_ids[i]), NULL, worker_thread_one_metric, &(para[i])); EXPECT_EQ(0, ret); } void *temp; for(int i = 0; i < n_thread; i++) { pthread_join(thread_ids[i], (void**)&temp); sum_durations += para[i].duration; //printf("Thread id:%d, operate counter:%d, duration:%lldus\n", i, n_loops, para[i].duration); } printf("Thread num:%d, operate counter:%d, duration:%lldus\n", n_thread, n_loops, sum_durations/n_thread); sleep(2); fieldstat_dynamic_instance_free(instance); } // TEST(FeildStatDynamicBenchmark, RecordExecTime) // { // int n_thread = 0; // int n_loops = 0; // for(int i = 0; i < 7; i++) // { // n_thread = 1 << i; // for(int j = 0; j < 9; j++) // { // n_loops = (int)pow(10, (double)j); // fieldstat_dynamic_benchmark(n_thread, n_loops); // } // } // } void build_Memory_benchmark(int n_thread, int n_metrics) { struct fieldstat_dynamic_instance *instance = NULL; // int n_metrics = 100000; // int n_thread = 1; int ret_n_metrics = 0; int ret_n_items = 0; int ret = 0; instance = fieldstat_dynamic_instance_new("firewall", n_thread); for(int i = 0; i < n_thread; i++) { ret_n_metrics = fieldstat_dynamic_read_metrics_cnt(instance, i); ret_n_items = fieldstat_dynamic_read_htable_item_cnt(instance, i); EXPECT_EQ(0, ret_n_metrics); EXPECT_EQ(0, ret_n_items); } for(int i = 0; i < n_thread; i++) { for(int j = 0; j < n_metrics; j++) { char metric_name[128] = {0}; memset(metric_name, 0, sizeof(metric_name)); snprintf(metric_name, sizeof(metric_name), "Active_sessions_%d", j); ret = fieldstat_dynamic_metric_value_incrby(instance, FIELD_METRIC_TYPE_GAUGE, metric_name, j, NULL, 0, i); EXPECT_EQ(0, ret); } } for(int i = 0; i < n_thread; i++) { ret_n_metrics = fieldstat_dynamic_read_metrics_cnt(instance, i); ret_n_items = fieldstat_dynamic_read_htable_item_cnt(instance, i); EXPECT_EQ(n_metrics, ret_n_metrics); EXPECT_EQ(n_metrics, ret_n_items); } struct rusage usage; ret = getrusage(RUSAGE_SELF, &usage); EXPECT_EQ(0, ret); printf("thread num:%2d, per thread metric num: %9d," "total metrics num:%9d, Max Resident Set Size: %ld KB\n", n_thread, n_metrics, n_thread * n_metrics, usage.ru_maxrss); fieldstat_dynamic_instance_free(instance); } TEST(FeildStatDynamicBenchmark, RecordExecTime) { int n_thread = 1; int n_metrics = 10000000; build_Memory_benchmark(n_thread, n_metrics); } int main(int argc, char *argv[]) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); }