bugfix:TSG-17151:修改fieldstat3单个metric占用内存较大的bugv3.0.16

2 files changed, 154 insertions, 109 deletions

diff --git a/test/src/gtest_dynamic_benchmark.cpp b/test/src/gtest_dynamic_benchmark.cpp
index b93df70..dc0379d 100644
--- a/test/src/gtest_dynamic_benchmark.cpp
+++ b/test/src/gtest_dynamic_benchmark.cpp
@@ -6,10 +6,11 @@
 #include "fieldstat_internal.h"
 #include "cJSON.h"
 #include <math.h>
+#include <sys/resource.h>
 
 /*
 date: 2023/9/5
-spinlock, output interval: 1ms
+spinlock, output interval: 1ms, Test case: RecordExecTime
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  |                 | Thread num 1           | Thread num 2           | Thread num 4           | Thread num 8           | Thread num 16          | Thread num 32          | Thread num 64          |
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
@@ -34,7 +35,7 @@ spinlock, output interval: 1ms
  | 100,000,000     | 19,425,541             | 30,764,254             | 57,262,235             | 97,336,038             | 117,999,613            | 156,962,386            | 203,782,206            |
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 
-no spinlock, output interval: 1ms
+no spinlock, output interval: 1ms, Test case: RecordExecTime
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  |                 | Thread num 1           | Thread num 2           | Thread num 4           | Thread num 8           | Thread num 16          | Thread num 32          | Thread num 64          |
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
@@ -59,7 +60,7 @@ no spinlock, output interval: 1ms
  | 100,000,000     | 18,784,181             | 18,827,600             | 18,812,896             | 18,875,007             | 18,833,475             | 18,811,591             | 26,965,032             |
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 
-spinlock with Struct Alignment, output interval: 1ms
+spinlock with Struct Alignment, output interval: 1ms, Test case: RecordExecTime
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  |                 | Thread num 1           | Thread num 2           | Thread num 4           | Thread num 8           | Thread num 16          | Thread num 32          | Thread num 64          |
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
@@ -89,7 +90,7 @@ spinlock with Struct Alignment, output interval: 1ms
 /*
 date: 2023/9/6
 
-no lock, output interval 1ms
+no lock, output interval 1ms, Test case: RecordExecTime
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  |                 | Thread num 1           | Thread num 2           | Thread num 4           | Thread num 8           | Thread num 16          | Thread num 32          | Thread num 64          |
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
@@ -114,7 +115,7 @@ no lock, output interval 1ms
  | 100,000,000     | 19,245,473             | 19,243,094             | 19,204,994             | 19,193,001             | 19,196,800             | 19,181,999             | 27,619,589             |
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 
-spinlock, output interval 1ms
+spinlock, output interval 1ms, Test case: RecordExecTime
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  |                 | Thread num 1           | Thread num 2           | Thread num 4           | Thread num 8           | Thread num 16          | Thread num 32          | Thread num 64          |
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
@@ -139,7 +140,7 @@ spinlock, output interval 1ms
  | 100,000,000     | 18,720,506             | 18,736,390             | 18,740,244             | 18,769,749             | 18,779,667             | 18,806,123             | 26,277,081             |
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 
-rwlock, output interval 1ms
+rwlock, output interval 1ms, Test case: RecordExecTime
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  |                 | Thread num 1           | Thread num 2           | Thread num 4           | Thread num 8           | Thread num 16          | Thread num 32          | Thread num 64          |
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
@@ -164,7 +165,7 @@ rwlock, output interval 1ms
  | 100,000,000     | 20,413,678             | 20,465,977             | 26,326,140             | 29,201,649             | 20,692,983             | 20,822,775             | 46,712,568             |
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 
-mutex, output interval 1ms
+mutex, output interval 1ms, Test case: RecordExecTime
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
  |                 | Thread num 1           | Thread num 2           | Thread num 4           | Thread num 8           | Thread num 16          | Thread num 32          | Thread num 64          |
  --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
@@ -191,6 +192,37 @@ mutex, output interval 1ms
 
 */
 
+/*
+date: 2023/9/20
+
+no lock, output interval 1ms, Test case: RecordExecTime
+ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ |                 | Thread num 1           | Thread num 2           | Thread num 4           | Thread num 8           | Thread num 16          | Thread num 32          | Thread num 64          |
+ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ | operate counter | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) | operation duration(us) |
+ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ | 1               | 28                     | 11                     | 11                     | 13					   | 28                     | 28					 | 25                     |
+ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ | 10              | 11                     | 8                      | 8                      | 8                      | 9                      | 7  					 | 6					  |
+ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ | 100             | 31                     | 28                     | 30                     | 25                     | 28                     | 25 					 | 24                     |
+ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ | 1,000           | 198                    | 200                    | 198                    | 198					   | 198                    | 198                    | 198                    |
+ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ | 10,000          | 1,878                  | 1,889                  | 1,882                  | 1,879                  | 1,886                  | 1,884                  | 1,882                  |
+ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ | 100,000         | 18,724                 | 18,627                 | 18,636				  | 18,678				   | 18,695                 | 19,995                 | 19,479                 |
+ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ | 1,000,000       | 187,366                | 186,860                | 186,783                | 186,636                | 186,446                | 188,050                | 239,472                |
+ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ | 10,000,000      | 1,865,582              | 1,863,954              | 1,862,493	          | 1,863,763              | 1,867,006              | 1,869,409              | 2,450,149              |
+ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ | 100,000,000     | 18,635,991             | 18,656,564             | 18,636,838             | 18,644,694             | 18,678,975             | 18,656,275             | 25,055,446             |
+ --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+*/
+
+
 struct thread_para
 {
 	int loops;
@@ -213,7 +245,7 @@ void _worker_thread_one_metric(void *arg)
 	int loops = para->loops;
 	int thread_id = para->thread_id;
 	struct fieldstat_dynamic_instance *instance = para->instance;
-	char metric_name[128] = {0};
+	//char metric_name[128] = {0};
 
 	int ret = 0;
 	long long start_time, end_time;
@@ -221,10 +253,10 @@ void _worker_thread_one_metric(void *arg)
 	start_time = current_timestamp();
 	for(int i = 0; i < loops; i++)
 	{
-		memset(metric_name, 0, sizeof(metric_name));
-		snprintf(metric_name, sizeof(metric_name), "Active_sessions_%d", i);
+		//memset(metric_name, 0, sizeof(metric_name));
+		//snprintf(metric_name, sizeof(metric_name), "Active_sessions_%d", i);
 		ret = fieldstat_dynamic_metric_value_incrby(instance, FIELD_TYPE_GAUGE,
-													metric_name, 10, 
+													"Active_sessions", 10, 
 													NULL, 0, thread_id); 
 		EXPECT_EQ(0, ret);
 	}
@@ -283,7 +315,9 @@ static void fieldstat_dynamic_benchmark(int n_thread, int n_loops)
 	fieldstat_dynamic_instance_free(instance);
 }
 
-// TEST(FeildStatDynamicAPI, NThread64Counter10Million)
+
+
+// TEST(FeildStatDynamicBenchmark, RecordExecTime)
 // {
 // 	int n_thread = 0;
 // 	int n_loops = 0;
@@ -299,23 +333,68 @@ static void fieldstat_dynamic_benchmark(int n_thread, int n_loops)
 // 	}
 // }
 
-TEST(FeildStatDynamicAPI, AllConditions)
+
+
+void build_Memory_benchmark(int n_thread, int n_metrics)
 {
-	int n_thread = 0;
-	int n_loops = 0;
+	struct fieldstat_dynamic_instance *instance = NULL;
+	// int n_metrics = 100000;
+	// int n_thread  = 1;
+
+	int ret_n_metrics = 0;
+	int ret_n_items   = 0;
+
+	int ret = 0;
+
+	instance = fieldstat_dynamic_instance_new("firewall", n_thread);
+
+	for(int i = 0; i < n_thread; i++)
+	{
+		ret_n_metrics = fieldstat_dynamic_read_metrics_cnt(instance, i);
+		ret_n_items   = fieldstat_dynamic_read_htable_item_cnt(instance,  i);
+		EXPECT_EQ(0, ret_n_metrics);
+		EXPECT_EQ(0, ret_n_items);
+	}
 
-	for(int i = 0; i < 5; i++)
+	for(int i = 0; i < n_thread; i++)
 	{
-		n_thread = 1 << i;
-		for(int j = 0; j < 6; j++)
+		for(int j = 0; j < n_metrics; j++)
 		{
-			n_loops = (int)pow(10, (double)j);
-			fieldstat_dynamic_benchmark(n_thread, n_loops);
+			char metric_name[128] = {0};
+			memset(metric_name, 0, sizeof(metric_name));
+			snprintf(metric_name, sizeof(metric_name), "Active_sessions_%d", j);
+
+			ret = fieldstat_dynamic_metric_value_incrby(instance, FIELD_TYPE_GAUGE, 
+														metric_name, j, NULL, 0, i); 
+			EXPECT_EQ(0, ret);
 		}
 	}
+
+	for(int i = 0; i < n_thread; i++)
+	{
+		ret_n_metrics = fieldstat_dynamic_read_metrics_cnt(instance, i);
+		ret_n_items   = fieldstat_dynamic_read_htable_item_cnt(instance,  i);
+		EXPECT_EQ(n_metrics, ret_n_metrics);
+		EXPECT_EQ(n_metrics, ret_n_items);
+	}
+
+    struct rusage usage;
+	ret = getrusage(RUSAGE_SELF, &usage);
+	EXPECT_EQ(0, ret);
+	printf("thread num:%2d, per thread metric num: %9d,"
+		   "total metrics num:%9d, Max Resident Set Size: %ld KB\n",
+		   n_thread, n_metrics, n_thread * n_metrics, usage.ru_maxrss);
+
+	fieldstat_dynamic_instance_free(instance);
 }
 
 
+TEST(FeildStatDynamicBenchmark, RecordExecTime)
+{
+	int n_thread  = 1;
+	int n_metrics = 10000000;
+	build_Memory_benchmark(n_thread, n_metrics);
+}
 
 int main(int argc, char *argv[]) 
 {
diff --git a/test/src/gtest_dynamic_fieldstat.cpp b/test/src/gtest_dynamic_fieldstat.cpp
index 6fc9d43..7313cde 100644
--- a/test/src/gtest_dynamic_fieldstat.cpp
+++ b/test/src/gtest_dynamic_fieldstat.cpp
@@ -2821,100 +2821,66 @@ TEST(FeildStatDynamicAPI, SendLenEqualUDPPayload)
 }
 
 
+TEST(FeildStatDynamicAPI, ReadPerThreadHtableInfo)
+{
+	struct fieldstat_dynamic_instance *instance = NULL;
+	int n_thread =  4;
+	int table_id = -1;
 
-// TEST(FeildStatDynamicAPI, SendTo)
-// {
-// 	unsigned int server_ip = 0;
-// 	int send_socket = startup_udp();
-// 	struct line_protocol_output output;
+	int ret_n_metrics = 0;
+	int ret_n_items   = 0;
 
-// 	inet_pton(AF_INET, "127.0.0.1", (void *)&(server_ip));
+	int ret = 0;
 
-// 	memset(&output, 0, sizeof(struct line_protocol_output));
-// 	output.send_socket = send_socket;
-// 	output.server_ip = server_ip;
-// 	output.server_port = 8700;
+	instance = fieldstat_dynamic_instance_new("firewall", n_thread);
+
+	enum field_type column_type[] = {FIELD_TYPE_COUNTER, FIELD_TYPE_COUNTER};
+	const char *column_name[]     = {"packages", "bytes"};
 	
+	unsigned int column_ids[2];
+	table_id = fieldstat_register_dynamic_table(instance, "tsg_master", column_name, 
+												column_type, 2, column_ids);
+	EXPECT_EQ(0, table_id);	
 
-// 	std::vector<std::string> send_buf;
-
-// 	char line_buf[1460];
-// 	int count = 0;
-
-// 	for(int i = 0; i < 90000; i++)
-// 	{
-// 		memset(line_buf, 0, sizeof(line_buf));
-// 		// snprintf(line_buf, sizeof(line_buf),
-// 		// "Active_session_%05d,app_name=firewall Active_session_01=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_02=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_03=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_04=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_05=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_06=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_07=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_08=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_09=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_10=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_11=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_12=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_13=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_14=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_15=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_16=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_17=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_18=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_19=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_20=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_21=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_22=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_23=12\n"
-// 		// "Active_session_%05d,app_name=firewall Active_session_24=12\n", 
-// 		// i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i);
-
-// 		snprintf(line_buf, sizeof(line_buf),
-// 		"Active_sessi01_%05d,app_name=firewall,Active_session_01=12,"
-// 		"Active_sessi02=%05d,app_name=firewa01,Active_session_02=12,"
-// 		"Active_sessi03=%05d,app_name=firewa02,Active_session_03=12,"
-// 		"Active_sessi04=%05d,app_name=firewa03,Active_session_04=12,"
-// 		"Active_sessi05=%05d,app_name=firewa04,Active_session_05=12,"
-// 		"Active_sessi06=%05d,app_name=firewa05,Active_session_06=12,"
-// 		"Active_sessi07=%05d,app_name=firewa06,Active_session_07=12,"
-// 		"Active_sessi08=%05d,app_name=firewa07,Active_session_08=12,"
-// 		"Active_sessi09=%05d,app_name=firewa08,Active_session_09=12,"
-// 		"Active_sessi10=%05d,app_name=firewa09,Active_session_10=12,"
-// 		"Active_sessi11=%05d,app_name=firewa10,Active_session_11=12,"
-// 		"Active_sessi12=%05d,app_name=firewa11,Active_session_12=12,"
-// 		"Active_sessi13=%05d,app_name=firewa12,Active_session_13=12,"
-// 		"Active_sessi14=%05d,app_name=firewa13,Active_session_14=12,"
-// 		"Active_sessi15=%05d,app_name=firewa14,Active_session_15=12,"
-// 		"Active_sessi16=%05d,app_name=firewa15,Active_session_16=12,"
-// 		"Active_sessi17=%05d,app_name=firewa16,Active_session_17=12,"
-// 		"Active_sessi18=%05d,app_name=firewa17,Active_session_18=12,"
-// 		"Active_sessi19=%05d,app_name=firewa18,Active_session_19=12,"
-// 		"Active_sessi20=%05d,app_name=firewa19,Active_session_20=12,"
-// 		"Active_sessi21=%05d,app_name=firewa20,Active_session_21=12,"
-// 		"Active_sessi22=%05d,app_name=firewa21,Active_session_22=12,"
-// 		"Active_sessi23=%05d,app_name=firewa22,Active_session_23=12,"
-// 		"Active_sessi24=%05d,app_name=firewa23 Active_session_24=12\n",
-// 		i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i, i);
-// 		send_buf.push_back(std::string(line_buf));  
-
-// 	}
-
-// 	system("cat /dev/null > /tmp/metrics.out");
-// 	for (auto it = send_buf.begin(); it != send_buf.end(); ++it) {
-// 		const std::string& str = *it;
-// 		//std::cout << str << " - Length: " << str.length() << std::endl;
-// 		//printf("%s\n", str.c_str());
-// 		send_udp(send_socket, server_ip, 8700, str.c_str(), str.length());
-// 		//printf("%5d,%s\n", (int)str.length(), str.c_str());
-// 		//send_line_buf(&output, str.c_str(), str.length());
-
-// 		count++;
-// 	}
-// 	printf("count:%d\n",count);
-
-// }
+	for(int i = 0; i < n_thread; i++)
+	{
+		ret_n_metrics = fieldstat_dynamic_read_metrics_cnt(instance, i);
+		ret_n_items   = fieldstat_dynamic_read_htable_item_cnt(instance,  i);
+		EXPECT_EQ(0, ret_n_metrics);
+		EXPECT_EQ(0, ret_n_items);
+	}
+
+	for(int i = 0; i < n_thread; i++)
+	{
+		ret = fieldstat_dynamic_metric_value_incrby(instance, FIELD_TYPE_GAUGE, 
+													"Active_sessions", 100, 
+													NULL, 0, i); 
+		EXPECT_EQ(0, ret);
+		ret = fieldstat_dynamic_metric_value_incrby(instance, FIELD_TYPE_COUNTER, 
+													"Traffic_bytes",   100, 
+													NULL, 0, i); 
+		EXPECT_EQ(0, ret);
+
+		ret = fieldstat_dynamic_table_metric_value_incrby(instance, table_id,
+														  column_ids[0], "row1",
+														  100, NULL, 0, i);
+
+		ret = fieldstat_dynamic_table_metric_value_incrby(instance, table_id,
+														  column_ids[0], "row2",
+														  100, NULL, 0, i);
+
+	}
+
+	for(int i = 0; i < n_thread; i++)
+	{
+		ret_n_metrics = fieldstat_dynamic_read_metrics_cnt(instance, i);
+		ret_n_items   = fieldstat_dynamic_read_htable_item_cnt(instance,  i);
+		EXPECT_EQ(6, ret_n_metrics);
+		EXPECT_EQ(4, ret_n_items);
+	}
+
+	fieldstat_dynamic_instance_free(instance);
+}
 
 
 int main(int argc, char *argv[])