Merge branch 'develop-version4' of git.mesalab.cn:MESA_framework/FieldStat into develop-version4

3 files changed, 204 insertions, 146 deletions

diff --git a/readme.md b/readme.md
index fd84af0..88e0e33 100644
--- a/readme.md
+++ b/readme.md
@@ -300,10 +300,11 @@ optional arguments:
   -i INTERVAL, --interval INTERVAL
                         interval, seconds to wait between print.
   -l, --loop            print loop, exit when recv a signal.
-  --clear-screen        clear screen at start of loop
-  --display-hll         Display hyperloglog type metrics
-  --display-hist        Display histogram type metrics
-  --display-counter     Display counter type metrics
+  --clear-screen        clear screen at start of loop.
+  --display-hll         Display hyperloglog type metrics.
+  --display-hist        Display histogram type metrics.
+  --display-counter     Display counter type metrics.
+  --disable-table       disable display table format.
   -m MATCH_TAGS, --match-tags MATCH_TAGS
                         Display the tags match metrics
 ```
diff --git a/src/exporter/fieldstat_exporter.py b/src/exporter/fieldstat_exporter.py
index 84f9a08..20a2c5a 100644
--- a/src/exporter/fieldstat_exporter.py
+++ b/src/exporter/fieldstat_exporter.py
@@ -62,6 +62,7 @@ class FieldstatExporterVars:
     local_display_hist    = False
     local_display_hll     = False
     local_match_tags      = {}
+    local_disable_table   = True
 
     prom_uri_path         = ""
 
@@ -228,33 +229,43 @@ class CounterTable:
         self.field_names = []  
         self.rows        = []
 
-    def add_field_names(self, fields):
-        if len(self.field_names) == 0:
-            sorted_keys = sorted(fields.keys())
-            self.field_names.append("")
-            self.field_names.extend(sorted_keys)
 
-    def append_field_rows(self,  tags, match_tags, fields):
-        row      = []
-        new_tags = {}
+    def create_row_table(self, fields):
+        field_names = []
+        sorted_keys = sorted(fields.keys())
+        field_names.append("")
+        field_names.extend(sorted_keys)
+
+        table = PrettyTable()
+        table.vrules = NONE
+        table.hrules = NONE
+        table.field_names = field_names
+
+        for item in field_names:
+            table.align[item] = "r"
+        table.align[""] = "l"
+
+        self.tables.append(table)
+
+        return table
 
-        #table only one tags
-        if len(tags) - len(match_tags) != 1:
-            return
 
-        for key in tags:
-            if key not in match_tags:
-                new_tags[key] = tags[key]
-                break
+    def add_row_table_row(self, table, tags, fields):
+        row = []
 
-        for key, value in new_tags.items():
+        if table is None:
+            return
+
+        #exporter table row name.
+        for key, value in tags.items():
             row.append("%s_%s" % (key, str(value)))
 
+        #exporter table row value.
         sorted_keys = sorted(fields.keys())
         for key in sorted_keys:
             row.append(fields[key])
 
-        self.rows.append(row)
+        table.add_row(row)
 
 
     def add_table_column(self, tags, head, value, speed_s):
@@ -299,23 +310,6 @@ class CounterTable:
                 table  = self.__build_one_table(self.columns[l_edge:r_edge])
             
             self.tables.append(table)
-        
-    def __build_rows_tables(self):
-        if len(self.field_names) == 0 or len(self.rows) == 0:
-            return
-        table = PrettyTable()
-        table.vrules = NONE
-        table.hrules = NONE
-        table.field_names = self.field_names
-        
-        for item in self.field_names:
-            table.align[item] = "r"
-        table.align[""] = "l"
-
-        for row in self.rows:
-            table.add_row(row)
-
-        self.tables.append(table)
 
     def read_columns_num(self):
         return len(self.columns)
@@ -328,7 +322,6 @@ class CounterTable:
 
     def print_tables(self):
         self.__build_columns_tables()
-        self.__build_rows_tables()
         for item in self.tables:
             print(item)
 
@@ -399,7 +392,7 @@ class LocalExporter:
         self.display_hist      = FieldstatExporterVars.local_display_hist
         self.display_hll       = FieldstatExporterVars.local_display_hll
         self.match_tags        = FieldstatExporterVars.local_match_tags
-        self.is_counter_table  = False
+        self.disable_table     = FieldstatExporterVars.local_disable_table
         self.__set_default_display()
 
     def __set_default_display(self):
@@ -452,82 +445,109 @@ class LocalExporter:
             
         return True
 
-    #table: one same tags + same fields keys 
-    def __is_counter_table(self, json_list):
-        is_first_elem = True
-        prev_tags     = {}
-        prev_fields   = {}
 
-        #only one counter type json object, counter type print normal mode
-        if len(json_list) <= 1:
-            return False
-
-        for json_object in json_list:
-            tags = copy.deepcopy(json_object["tags"])
+    def __generate_table_bundle(self, json_objects):
+        table_bundle = {}
+        table_bundle["not_table_field"] = []
 
-            if not self.__match_tags(tags):
+        for item in json_objects:
+            #set display table option off
+            if self.disable_table:
+                table_bundle["not_table_field"].append(item)
+                continue
+            # table: only one tag and same tag key + same field keys + name key
+            not_append_table = False
+            if len(item["tags"]) != 1:
+                table_bundle["not_table_field"].append(item)
                 continue
 
-            for _, value in json_object["fields"].items():
+            for _, value in item["fields"].items():                
                 if isinstance(value, str):
-                    return False
+                    not_append_table = True
+                    break
 
-            for key in self.match_tags:
-                tags.pop(key, None)
+            if not_append_table == True:
+                table_bundle["not_table_field"].append(item)
+                continue
+
+            key_list  = list(item["tags"].keys()) + sorted(list(item["fields"].keys()))
+            key = ''.join(key_list) + item["name"]
+            
+            if key in table_bundle:
+                val = table_bundle[key]
+                val.append(item)
+            else:
+                table_bundle[key] = [item]
+
+        return table_bundle
 
-            #fields no one tags. print normal mode.
-            if len(tags) != 1:
-                return False
 
-            if is_first_elem:
-                prev_tags     = tags
-                prev_fields   = json_object["fields"]
-                is_first_elem = False
+    def __read_match_tags_objects(self, json_objects):
+        match_objects = []
+
+        for item in json_objects:
+            tags = item["tags"]
+            #not match tags object. not read.
+            if not self.__match_tags(tags):
                 continue
-            else:
-                if prev_tags.keys()   == tags.keys() and \
-                   prev_fields.keys() == json_object["fields"].keys():
-                   continue
-                else:
-                    return False
-        
-        return True
+            #match tags object. delete matching tags.
+            for key,value in self.match_tags.items():
+                if key in tags and value == tags[key]:
+                    tags.pop(key, None)
+
+            match_objects.append(item)
 
+        return match_objects
 
-    def __parse_json_object(self, json_object):
+    def __parse_table_json_object(self, json_objects):
+        table = None
+        for item in json_objects:
+            tags   = item["tags"]
+            fields = item["fields"]
+            if table == None:
+                table = self.ctable.create_row_table(fields)
+            self.ctable.add_row_table_row(table, tags, fields)
+
+    def __parse_single_json_object(self, json_object):
         tags   = self.__parse_json_tags(json_object)
         fields = json_object["fields"]
 
-        if not self.__match_tags(json_object["tags"]):
-            return
-
         for key,value in fields.items():
             if not isinstance(value, str):
+                #counter type 
                 if key.endswith("_delta"):
                     continue
                 speed_s = self.__get_counter_speed_value(key, fields, json_object)
-                if self.is_counter_table:
-                    self.ctable.add_field_names(fields)
-                    self.ctable.append_field_rows(json_object["tags"], self.match_tags, fields)
-                    continue
-                else:
-                    self.__dealwith_counter(tags, key, value, speed_s)
+                self.__dealwith_counter(tags, key, value, speed_s)
             else:
+                # histogram and hll type
                 is_hll = FieldstatAPI.libfieldstat.fieldstat_is_hll(value.encode('utf-8'))
                 if is_hll:
                     self.__dealwith_hll(tags, key, value)
                 else:
                     self.__dealwith_histogram(tags, key, value)
 
+
     def parse_data(self):
+        #check source json file is exist.
         if not os.path.exists(self.json_path):
             logging.error("Path: {%s} does not exist", self.json_path)
             return
+
         with open(self.json_path) as file:
-            data = json.load(file)
-            self.is_counter_table = self.__is_counter_table(data)
-            for json_object in data:
-                self.__parse_json_object(json_object)
+            json_objects  = json.load(file)
+            #read match tags objects.
+            match_objects = self.__read_match_tags_objects(json_objects)
+            #generate tables dict.
+            table_bundle  = self.__generate_table_bundle(match_objects)
+
+            for tkey,tval in table_bundle.items():
+                if tkey == "not_table_field": # exporter single metrics.
+                    for item in tval:
+                        self.__parse_single_json_object(item)
+                else:                         # exporter table-format metrics.
+                    self.__parse_table_json_object(tval)
+
 
     def __print_top_edge(self):
         timestamp = datetime.datetime.now().timestamp()
@@ -615,6 +635,8 @@ class FieldstatExporter:
                             help = 'Display histogram type metrics')
         parser.add_argument('--display-counter', action = 'store_true', default = False,
                             help = 'Display counter type metrics')
+        parser.add_argument('--disable-table',   action = 'store_true', default = False,
+                            help = 'disable display table format')
         parser.add_argument("-m", "--match-tags", type = str,           default = "",
                             help = "Display the tags match metrics")
 
@@ -660,6 +682,7 @@ class FieldstatExporter:
             FieldstatExporterVars.local_display_hll     = args.display_hll
             FieldstatExporterVars.local_display_hist    = args.display_hist
             FieldstatExporterVars.local_match_tags      = self.__parse_tags_str(args.match_tags)
+            FieldstatExporterVars.local_disable_table   = args.disable_table
             self.exporter_mode       = 'local'
             self.local_interval_s    = args.interval
             self.local_enable_loop   = args.loop
diff --git a/test/test_fieldstat_exporter.py b/test/test_fieldstat_exporter.py
index 3a68512..562af4b 100644
--- a/test/test_fieldstat_exporter.py
+++ b/test/test_fieldstat_exporter.py
@@ -259,30 +259,30 @@ class TestCounterTable(unittest.TestCase):
     def setUp(self):
         self.c_table = CounterTable()
 
-
-    def test_add_field_names(self):
-        self.c_table.field_names = []
-
+    def test_create_row_table(self):
         fields_names_0 = {"column0": 0, "column1": 1}
-        self.c_table.add_field_names(fields_names_0)
-        self.assertEqual(self.c_table.field_names, ["", "column0", "column1"])
+        table = self.c_table.create_row_table(fields_names_0)
+        self.assertEqual(table.field_names, ["", "column0", "column1"])
 
         fields_names_1 = {"column2": 2, "column3": 3}
-        self.c_table.add_field_names(fields_names_1)
-        self.assertEqual(self.c_table.field_names, ["", "column0", "column1"])
+        table = self.c_table.create_row_table(fields_names_1)
+        self.assertEqual(table.field_names, ["", "column2", "column3"])
 
-
-    def test_append_field_rows(self):
+    def test_add_row_table_row(self):
         self.c_table.field_names = []
 
-        fields_names_0 = {"column0": 0, "column1": 1}
-        self.c_table.append_field_rows({"name": "0"}, {}, fields_names_0)
-        self.assertEqual(self.c_table.rows[0], ["name_0", 0, 1])
-
-        fields_names_1 = {"column2": 2, "column3": 3}
-        self.c_table.append_field_rows({"name": "1"}, {}, fields_names_1)
-        self.assertEqual(self.c_table.rows[1],  ["name_1", 2, 3])
+        table = self.c_table.create_row_table({"column0": 0, "column1": 1})
+        tags  = {"row": "0"}
+        field = {"column0": 0, "column1":1}
+        self.c_table.add_row_table_row(None,  tags, field)
 
+        table_str = table.get_string()
+        row_count = len(table_str.split("\n")) - 1
+        self.assertEqual(row_count, 0)
+        
+        self.c_table.add_row_table_row(table, tags, field)
+        row_count = len(table_str.split("\n")) - 1
+        self.assertEqual(row_count, 0)
 
     def test_add_table_column(self):
         head    = "policy_hit"
@@ -325,16 +325,6 @@ class TestCounterTable(unittest.TestCase):
             self.assertEqual(self.c_table.read_tables_num(), math.ceil(100/table_size))
 
 
-    def test__build_rows_tables(self):
-        self.c_table.field_names = ["", "column0", "column1"]
-        self.c_table.rows = [["row0", 1, 1], ["row1", 2, 2]]
-        
-        n_tables_before = self.c_table.read_tables_num()
-        self.c_table._CounterTable__build_rows_tables()
-        n_table_after   = self.c_table.read_tables_num()
-        self.assertEqual(n_table_after - n_tables_before, 1)
-
-
     def test_print_tables(self):
         self.c_table.columns = []
         for i in range(100):
@@ -572,7 +562,7 @@ class TestLocalExporter(unittest.TestCase):
         ret = self.local._LocalExporter__match_tags(tags)
         self.assertEqual(ret, True)
 
-    def test__is_counter_table(self):
+    def test__generate_table_bundle(self):
         json_data_0 = [{ "name": "-",
                         "tags": {
                             "send_log": "sum"
@@ -584,8 +574,9 @@ class TestLocalExporter(unittest.TestCase):
                         "timestamp_ms": 1000,
                         "timestamp_ms_delta": 0
                         }]
-        ret = self.local._LocalExporter__is_counter_table(json_data_0)
-        self.assertEqual(ret, False)
+        table_bundle_0 = self.local._LocalExporter__generate_table_bundle(json_data_0)
+        self.assertEqual(table_bundle_0["not_table_field"], [])
+        self.assertEqual(len(table_bundle_0["send_logT_fail_logT_fail_log_delta-"]), 1)
 
         json_data_1 = [{ "name": "-",
                         "tags": {
@@ -609,14 +600,9 @@ class TestLocalExporter(unittest.TestCase):
                         "timestamp_ms": 1000,
                         "timestamp_ms_delta": 0
                         }]
-        
-        self.local.match_tags = {"policy": 1} 
-        ret = self.local._LocalExporter__is_counter_table(json_data_1)
-        self.assertEqual(ret, True)
-        
-        self.local.match_tags = {"send_log": "sum"} 
-        ret = self.local._LocalExporter__is_counter_table(json_data_1)
-        self.assertEqual(ret, False)
+        table_bundle_1 = self.local._LocalExporter__generate_table_bundle(json_data_1)
+        self.assertEqual(table_bundle_1["not_table_field"], [])
+        self.assertEqual(len(table_bundle_1["send_logT_fail_logT_fail_log_delta-"]), 2)
 
         json_data_2 = [{ "name": "-",
                         "tags": {
@@ -640,9 +626,10 @@ class TestLocalExporter(unittest.TestCase):
                         "timestamp_ms": 1000,
                         "timestamp_ms_delta": 0
                         }]
-        self.local.match_tags = {} 
-        ret = self.local._LocalExporter__is_counter_table(json_data_2)
-        self.assertEqual(ret, False)
+        table_bundle_2 = self.local._LocalExporter__generate_table_bundle(json_data_2)
+        self.assertEqual(table_bundle_2["not_table_field"], [])
+        self.assertEqual(len(table_bundle_2["send_logT_fail_logT_fail_log_delta-"]), 1)
+        self.assertEqual(len(table_bundle_2["send_log_0T_fail_logT_fail_log_delta-"]), 1)
 
         json_data_3 = [{ "name": "-",
                         "tags": {
@@ -666,26 +653,72 @@ class TestLocalExporter(unittest.TestCase):
                         "timestamp_ms": 1000,
                         "timestamp_ms_delta": 0
                         }]
-        self.local.match_tags = {} 
-        ret = self.local._LocalExporter__is_counter_table(json_data_3)
-        self.assertEqual(ret, True)
+        table_bundle_3 = self.local._LocalExporter__generate_table_bundle(json_data_3)
+        self.assertEqual(table_bundle_3["not_table_field"], [])
+        self.assertEqual(len(table_bundle_3["send_logT_fail_logT_fail_log_delta-"]), 2)
+
+        json_data_4 = [{ "name": "-",
+                        "tags": {
+                            "send_log": "sum"
+                        },
+                        "fields": {
+                            "T_fail_log": 2,
+                            "T_fail_log_delta": 2
+                        },
+                        "timestamp_ms": 1000,
+                        "timestamp_ms_delta": 0
+                        },
+                        { "name": "-",
+                        "tags": {
+                            "send_log": "firewall"
+                        },
+                        "fields": {
+                            "T_fail_log": 2,
+                            "T_fail_log_delta": 2
+                        },
+                        "timestamp_ms": 1000,
+                        "timestamp_ms_delta": 0
+                        }]
+        self.local.disable_table = True
+        table_bundle_4 = self.local._LocalExporter__generate_table_bundle(json_data_4)
+        self.assertEqual(len(table_bundle_4["not_table_field"]), 2)
+
+    def test__read_match_tags_objects(self):
+        json_objects = [{ "name": "-",
+                        "tags": {
+                            "send_log": "sum"
+                        },
+                        "fields": {
+                            "T_fail_log": 2,
+                            "T_fail_log_delta": 2
+                        },
+                        "timestamp_ms": 1000,
+                        "timestamp_ms_delta": 0
+                        },
+                        { "name": "-",
+                        "tags": {
+                            "send_log": "firewall"
+                        },
+                        "fields": {
+                            "T_fail_log": 2,
+                            "T_fail_log_delta": 2
+                        },
+                        "timestamp_ms": 1000,
+                        "timestamp_ms_delta": 0
+                        }]
+
+        self.local.match_tags = {}
+        match_object = self.local._LocalExporter__read_match_tags_objects(json_objects)
+        self.assertEqual(len(match_object), 2)
+    
+        self.local.match_tags = {"test": 1}
+        match_object = self.local._LocalExporter__read_match_tags_objects(json_objects)
+        self.assertEqual(len(match_object), 0)
+
+        self.local.match_tags = {"send_log": "firewall"}
+        match_object = self.local._LocalExporter__read_match_tags_objects(json_objects)
+        self.assertEqual(len(match_object), 1)
 
-    def test__parse_json_object(self):
-        #counter json object
-        peradd  = len(self.local.ctable.columns)
-        self.local._LocalExporter__parse_json_object(self.counter_json_object)
-        postadd = len(self.local.ctable.columns) 
-        self.assertEqual(postadd - peradd, 1)
-         #histogram json object
-        peradd  = len(self.local.htable.tables)
-        self.local._LocalExporter__parse_json_object(self.hist_json_object)
-        postadd = len(self.local.htable.tables) 
-        self.assertEqual(postadd - peradd, 1)
-        #hll json object
-        peradd  = len(self.local.hlltable.columns)
-        self.local._LocalExporter__parse_json_object(self.hll_json_object)
-        postadd = len(self.local.hlltable.columns) 
-        self.assertEqual(postadd - peradd, 1)
 
     def test_parse_data(self):
         self.local.hlltable = CounterTable()
@@ -750,13 +783,14 @@ class TestFieldstatExporter(unittest.TestCase):
         self.assertEqual(args.loop,         False)
         self.assertEqual(args.clear_screen, False)
 
-        args = parser.parse_args(["local", "--loop", "--clear-screen", "-i", "1000", "--display-hist", "--display-hll", "--display-counter", "--match-tags", "policy:1,rule:1"])
+        args = parser.parse_args(["local", "--loop", "--clear-screen", "-i", "1000", "--display-hist", "--display-hll", "--display-counter", "--disable-table", "--match-tags", "policy:1,rule:1"])
         self.assertEqual(args.interval,        1000)
         self.assertEqual(args.loop,            True)
         self.assertEqual(args.clear_screen,    True)
         self.assertEqual(args.display_counter, True)
         self.assertEqual(args.display_hist,    True)
         self.assertEqual(args.display_hll,     True)
+        self.assertEqual(args.disable_table,   True)
         self.assertEqual(args.match_tags,      "policy:1,rule:1")