feature:TSG-17564:Adapt to json format changes and add template argument.v4.4.1

1 files changed, 339 insertions, 198 deletions

diff --git a/src/exporter/fieldstat_exporter.py b/src/exporter/fieldstat_exporter.py
index 20a2c5a..cdb7f1d 100644
--- a/src/exporter/fieldstat_exporter.py
+++ b/src/exporter/fieldstat_exporter.py
@@ -10,8 +10,10 @@ import shutil
 import datetime
 import time
 import copy
+import fcntl
 from prettytable import PrettyTable,NONE,HEADER
 from http.server import HTTPServer, BaseHTTPRequestHandler
+from jinja2 import Environment, FileSystemLoader
 
 import ctypes
 
@@ -62,7 +64,7 @@ class FieldstatExporterVars:
     local_display_hist    = False
     local_display_hll     = False
     local_match_tags      = {}
-    local_disable_table   = True
+    local_template        = ""
 
     prom_uri_path         = ""
 
@@ -158,8 +160,6 @@ class PrometheusExporter:
         for key,value in json_dict["fields"].items():
             escaped_name = self.__escape_metric_name(key)
             if isinstance(value, int):
-                if key.endswith("_delta"):
-                    continue
                 metrics += self.__build_type_counter(escaped_name, escaped_tags, value)
             else:
                 is_hll = FieldstatAPI.libfieldstat.fieldstat_is_hll(value.encode('utf-8'))
@@ -177,7 +177,9 @@ class PrometheusExporter:
             return payload
 
         with open(self.json_path) as file:
+            fcntl.flock(file, fcntl.LOCK_EX)
             json_data = json.load(file)
+            fcntl.flock(file, fcntl.LOCK_UN)
             for item in json_data:
                 payload += self.__build_metrics(item)
 
@@ -225,47 +227,7 @@ class CounterTable:
         self.min_width   = self.INFO_COLUMN_WIDTH
         self.tables      = []
         self.columns     = []
-        self.rows        = []
         self.field_names = []  
-        self.rows        = []
-
-
-    def create_row_table(self, fields):
-        field_names = []
-        sorted_keys = sorted(fields.keys())
-        field_names.append("")
-        field_names.extend(sorted_keys)
-
-        table = PrettyTable()
-        table.vrules = NONE
-        table.hrules = NONE
-        table.field_names = field_names
-
-        for item in field_names:
-            table.align[item] = "r"
-        table.align[""] = "l"
-
-        self.tables.append(table)
-
-        return table
-
-
-    def add_row_table_row(self, table, tags, fields):
-        row = []
-
-        if table is None:
-            return
-
-        #exporter table row name.
-        for key, value in tags.items():
-            row.append("%s_%s" % (key, str(value)))
-
-        #exporter table row value.
-        sorted_keys = sorted(fields.keys())
-        for key in sorted_keys:
-            row.append(fields[key])
-
-        table.add_row(row)
 
 
     def add_table_column(self, tags, head, value, speed_s):
@@ -287,7 +249,7 @@ class CounterTable:
 
         return table
     
-    def __build_columns_tables(self):
+    def build_columns_tables(self):
         # One table print in screen size
         # One table per screen size
         n_columns  = len(self.columns)
@@ -297,7 +259,6 @@ class CounterTable:
         table_size = self.column_size // (self.min_width + self.COLUMM_PADDING)
         if 0 == table_size:
             table_size = 1
-        
 
         for i in range(0, n_columns, table_size):
             table  = None
@@ -321,56 +282,134 @@ class CounterTable:
         return len(self.tables)
 
     def print_tables(self):
-        self.__build_columns_tables()
+        self.build_columns_tables()
         for item in self.tables:
             print(item)
 
 
-class HistogramTable:
+class TableFormatTable:
     def __init__(self):
-        self.format = FieldstatExporterVars.hist_format
-        self.bins   = FieldstatExporterVars.hist_bins
         self.tables = []
+        self.rows   = []
 
-    def __build_summary_format(self, c_hist, table):
-        for i in self.bins:
-            head = "{:.2f}%".format(i * 100)
-            row  =  FieldstatAPI.libfieldstat.fieldstat_histogram_value_at_percentile(c_hist, float(i * 100))
-            table.add_column(head, [row])
+    def create_table(self, fields):
+        field_names = []
+        sorted_keys = sorted(fields.keys())
+        field_names.append("")
+        field_names.extend(sorted_keys)
 
-    def __build_histogram_format(self, c_hist, table):
-        for i in self.bins:
-            head = "le={:d}".format(int(i))
-            row  =  FieldstatAPI.libfieldstat.fieldstat_histogram_value_at_percentile(c_hist, int(i))
-            table.add_column(head, [row])
+        table = PrettyTable()
+        table.vrules = NONE
+        table.hrules = NONE
+        table.field_names = field_names
+
+        for item in field_names:
+            table.align[item] = "r"
+        table.align[""] = "l"
+
+        self.tables.append(table)
+
+        return table
+
+
+    def add_table_row(self, table, tags, fields):
+        row = []
+
+        if table is None:
+            return
+
+        #exporter table row name.
+        for key, value in tags.items():
+            row.append("%s_%s" % (key, str(value)))
+
+        #exporter table row value.
+        sorted_keys = sorted(fields.keys())
+        for key in sorted_keys:
+            row.append(fields[key])
+
+        table.add_row(row)
+
+
+    def print_tables(self):
+        for item in self.tables:
+            print(item)
 
 
-    def build_table(self, tags, key, value):
-        table  = PrettyTable()
-        c_hist = FieldstatAPI.libfieldstat.fieldstat_histogram_base64_decode(value.encode('utf-8'))
-        if self.format == "summary":
-            self.__build_summary_format(c_hist, table)
-        if self.format == "histogram":
-            self.__build_histogram_format(c_hist, table)
+class HistogramTable:
+    def __init__(self):
+        self.format = FieldstatExporterVars.hist_format
+        self.bins   = FieldstatExporterVars.hist_bins
+        self.tables = []
+
+    def __get_row_shared_values(self, c_hist):
+        shared_values = []
 
         max_value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_max(c_hist)
-        table.add_column("MAX", [max_value])
-        
+        shared_values.append(str(max_value))
+
         min_value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_min(c_hist)
-        table.add_column("MIN", [min_value])
-        
+        shared_values.append(str(min_value))
+
         avg_value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_mean(c_hist)
-        table.add_column("AVG", ["{:.2f}".format(avg_value)])
-        
+        shared_values.append("{:.2f}".format(avg_value))
+
         dev_value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_stddev(c_hist)
-        table.add_column("STDDEV", ["{:.2f}".format(dev_value)])
+        shared_values.append("{:.2f}".format(dev_value))
         
         cnt_value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_total_count(c_hist)
-        table.add_column("CNT", [cnt_value])
+        shared_values.append(str(cnt_value))
+        
+        return shared_values
+
+    def  __get_row_values(self, c_hist):
+        row_values = []
+        for i in self.bins:
+            if self.format == "summary":
+                value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_at_percentile(c_hist, float(i * 100))
+                row_values.append(str(value))
+            if self.format == "histogram":
+                value = FieldstatAPI.libfieldstat.fieldstat_histogram_count_le_value(c_hist, int(i))
+                row_values.append(str(value))
+        shared_values = self.__get_row_shared_values(c_hist)
+        row_values   += shared_values
+        return row_values
+
+    def  __add_table_field_names(self, table):
+        field_names = []
+        field_names.append("")
+        for i in self.bins:
+            if self.format == "summary":
+                field_names.append("{:.2f}%".format(i * 100))
+            if self.format == "histogram":
+                field_names.append("le={:d}".format(int(i)))
+        field_names += ["MAX", "MIN", "AVG", "STDDEV", "CNT"]
+        table.field_names = field_names
+        for item in field_names:
+            table.min_width[item] = 10
+            if item == "":
+                table.align[item] = "l"
+            else:
+                table.align[item] = "r"
 
+    def __add_table_row(self, table, hist_value, row_name):
+        if hist_value is None:
+            return
+        c_hist = FieldstatAPI.libfieldstat.fieldstat_histogram_base64_decode(hist_value.encode('utf-8'))
+        row    = [row_name] + self.__get_row_values(c_hist)
         FieldstatAPI.libfieldstat.fieldstat_histogram_free(c_hist)
+        table.add_row(row)
+
+    def build_table(self, tags, key, hist_value, hist_value_delta, interval_ms):
+        table = PrettyTable()
+        table.vrules = NONE
+        # table.hrules = NONE
+        table.title = key + " " + tags + " "
+        if interval_ms > 0:
+            table.title += "interval: {:d}ms".format(int(interval_ms))
+        self.__add_table_field_names(table)
+        self.__add_table_row(table, hist_value, "acc")
+        self.__add_table_row(table, hist_value_delta, "delta")
 
-        table.title = key + " " + tags
         self.tables.append(table)
 
     def read_tables_num(self):
@@ -388,46 +427,44 @@ class LocalExporter:
         self.ctable            = CounterTable()
         self.htable            = HistogramTable()
         self.hlltable          = CounterTable()
+        self.tftable           = TableFormatTable()
         self.display_counter   = FieldstatExporterVars.local_display_counter
         self.display_hist      = FieldstatExporterVars.local_display_hist
         self.display_hll       = FieldstatExporterVars.local_display_hll
         self.match_tags        = FieldstatExporterVars.local_match_tags
-        self.disable_table     = FieldstatExporterVars.local_disable_table
+        self.template          = FieldstatExporterVars.local_template
         self.__set_default_display()
+        self.objects_matched   = []
+        self.template_ja2      = None 
+
 
     def __set_default_display(self):
-        #default print all type metrics
         if not (self.display_counter or self.display_hist or self.display_hll):
             self.display_counter = True
             self.display_hist    = True
             self.display_hll     = True
 
-    def __dealwith_counter(self, tags, key, value, speed_s):
+
+    def __build_counter_type_exporter(self, tags, key, value, value_delta, timestamp_ms_delta):
+        speed_s = 0.0
+        if value_delta is not None:
+            speed_s = value_delta * 1000 / timestamp_ms_delta
         self.ctable.add_table_column(tags, key, value, speed_s)
 
-    def __dealwith_histogram(self, tags, key, value):
-        self.htable.build_table(tags, key, value)
+    def __build_histogram_type_exporter(self, tags, key, value, value_delta, timestamp_ms_delta):
+        self.htable.build_table(tags, key, value, value_delta, timestamp_ms_delta)
 
-    def __dealwith_hll(self, tags, key, value):
+    def __build_hll_type_exporter(self, tags, key, value):
         hll_value = FieldstatAPI.libfieldstat.fieldstat_hll_base64_to_count(value.encode('utf-8'))
         self.hlltable.add_table_column(tags, key, "{:.2f}".format(hll_value), 0)
 
-    def __parse_json_tags(self, json_object):
+    def __append_app_name_to_tags(self, json_object):
         tags_dict = copy.deepcopy(json_object["tags"])
         tags_dict.update({"app_name": json_object["name"]})
         return json.dumps(tags_dict)
 
-    def __get_counter_speed_value(self, key, fields, json_object):
-        delta_key = key + "_delta"
-        if delta_key in fields:
-            delta_val = fields[delta_key]
-            delta_ms  = json_object["timestamp_ms"] - json_object["timestamp_ms_delta"]
-            speed_s   = delta_val * 1000 / delta_ms
-            return speed_s
-        return 0
-
 
-    def __match_tags(self, tags):
+    def __is_tags_matched(self, tags):
         if len(self.match_tags) == 0:
             return True
 
@@ -442,146 +479,250 @@ class LocalExporter:
                 return False
             if value != tags[key]:
                 return False
-            
-        return True
-
 
-    def __generate_table_bundle(self, json_objects):
-        table_bundle = {}
-        table_bundle["not_table_field"] = []
-
-        for item in json_objects:
-            #set display table option off
-            if self.disable_table:
-                table_bundle["not_table_field"].append(item)
-                continue
-            # table: only one tag and same tag key + same field keys + name key
-            not_append_table = False
-            if len(item["tags"]) != 1:
-                table_bundle["not_table_field"].append(item)
-                continue
-
-            for _, value in item["fields"].items():                
-                if isinstance(value, str):
-                    not_append_table = True
-                    break
-
-            if not_append_table == True:
-                table_bundle["not_table_field"].append(item)
-                continue
+        return True
 
-            key_list  = list(item["tags"].keys()) + sorted(list(item["fields"].keys()))
-            key = ''.join(key_list) + item["name"]
-            
-            if key in table_bundle:
-                val = table_bundle[key]
-                val.append(item)
-            else:
-                table_bundle[key] = [item]
 
-        return table_bundle
+    def read_json_objects_from_file(self):
+        #check source json file is exist.
+        objects = []
+        if not os.path.exists(self.json_path):
+            logging.error("Path: {%s} does not exist", self.json_path)
+            return objects
+        with open(self.json_path) as fd:
+            fcntl.flock(fd, fcntl.LOCK_EX)
+            objects = json.load(fd)
+            fcntl.flock(fd, fcntl.LOCK_UN)
+        return objects
 
 
-    def __read_match_tags_objects(self, json_objects):
-        match_objects = []
+    def read_match_tags_json_objects(self, json_objects):
+        matched_objects = []
 
         for item in json_objects:
             tags = item["tags"]
             #not match tags object. not read.
-            if not self.__match_tags(tags):
+            if not self.__is_tags_matched(tags):
                 continue
             #match tags object. delete matching tags.
             for key,value in self.match_tags.items():
                 if key in tags and value == tags[key]:
                     tags.pop(key, None)
 
-            match_objects.append(item)
+            matched_objects.append(item)
+
+        self.objects_matched = matched_objects
+
+        return matched_objects
+
+    # def build_table_format_htable(self, json_objects):
+    #     table_bundle = {}
+    #     table_bundle["not_table_format"] = []
+
+    #     for item in json_objects:
+    #         #set display table option off
+    #         if self.disable_table:
+    #             table_bundle["not_table_format"].append(item)
+    #             continue
+    #         # table: only one tag and same tag key + same field keys + name key
+    #         not_append_table = False
+    #         if len(item["tags"]) != 1:
+    #             table_bundle["not_table_format"].append(item)
+    #             continue
+
+    #         for _, value in item["fields"].items():                
+    #             if isinstance(value, str):
+    #                 not_append_table = True
+    #                 break
 
-        return match_objects
+    #         if not_append_table == True:
+    #             table_bundle["not_table_format"].append(item)
+    #             continue
 
-    def __parse_table_json_object(self, json_objects):
+    #         key_list  = list(item["tags"].keys()) + sorted(list(item["fields"].keys()))
+    #         key = ''.join(key_list) + item["name"]
+            
+    #         if key in table_bundle:
+    #             val = table_bundle[key]
+    #             val.append(item)
+    #         else:
+    #             table_bundle[key] = [item]
+
+    #     return table_bundle
+
+
+    def build_not_table_format_exporter(self, json_objects):
+        for item in json_objects:
+            timestamp_ms_delta = item["timestamp_ms_delta"]
+            fields_delta       = item["fields_delta"]
+            tags_new           = self.__append_app_name_to_tags(item)
+
+            for key,value in item["fields"].items():
+                value_delta = None
+                if key in fields_delta:
+                    value_delta = fields_delta[key]
+                if not isinstance(value, str):
+                    self.__build_counter_type_exporter(tags_new, key, value, value_delta, timestamp_ms_delta)
+                else:
+                    # histogram and hll type
+                    is_hll = FieldstatAPI.libfieldstat.fieldstat_is_hll(value.encode('utf-8'))
+                    if is_hll:
+                        self.__build_hll_type_exporter(tags_new, key, value)
+                    else:
+                        self.__build_histogram_type_exporter(tags_new, key, value, value_delta, timestamp_ms_delta)
+
+
+    def build_table_format_exporter(self, json_objects):
         table = None
         for item in json_objects:
             tags   = item["tags"]
             fields = item["fields"]
             if table == None:
-                table = self.ctable.create_row_table(fields)
-            self.ctable.add_row_table_row(table, tags, fields)
+                table = self.tftable.create_table(fields)
+            self.tftable.add_table_row(table, tags, fields)
+        return table
 
-    def __parse_single_json_object(self, json_object):
-        tags   = self.__parse_json_tags(json_object)
-        fields = json_object["fields"]
 
-        for key,value in fields.items():
-            if not isinstance(value, str):
-                #counter type 
-                if key.endswith("_delta"):
-                    continue
-                speed_s = self.__get_counter_speed_value(key, fields, json_object)
-                self.__dealwith_counter(tags, key, value, speed_s)
-            else:
-                # histogram and hll type
-                is_hll = FieldstatAPI.libfieldstat.fieldstat_is_hll(value.encode('utf-8'))
-                if is_hll:
-                    self.__dealwith_hll(tags, key, value)
-                else:
-                    self.__dealwith_histogram(tags, key, value)
+    def export_templates(self):
+        env = Environment(loader=FileSystemLoader('templates'))
+        env.globals.update(print_tables    =self.print_table_format)
+        env.globals.update(print_counters  =self.print_counter_type)
+        env.globals.update(print_histograms=self.print_histogram_type)
+        env.globals.update(print_hlls      =self.print_hll_type)
+        template = env.from_string(self.template)
+        self.template_ja2 = template
 
+    def build_local_exporter(self):
+        objects = self.read_json_objects_from_file()
+        self.objects_matched = self.read_match_tags_json_objects(objects)
 
-    def parse_data(self):
-        #check source json file is exist.
-        if not os.path.exists(self.json_path):
-            logging.error("Path: {%s} does not exist", self.json_path)
-            return
+        if len(self.template) > 0:
+            self.export_templates()
+        else:
+            self.build_not_table_format_exporter(self.objects_matched)
 
-        with open(self.json_path) as file:
-            json_objects  = json.load(file)
-            #read match tags objects.
-            match_objects = self.__read_match_tags_objects(json_objects)
-            #generate tables dict.
-            table_bundle  = self.__generate_table_bundle(match_objects)
-
-            for tkey,tval in table_bundle.items():
-                if tkey == "not_table_field": # exporter single metrics.
-                    for item in tval:
-                        self.__parse_single_json_object(item)
-                else:                         # exporter table-format metrics.
-                    self.__parse_table_json_object(tval)
-
-
-    def __print_top_edge(self):
-        timestamp = datetime.datetime.now().timestamp()
-        formatted_time = datetime.datetime.fromtimestamp(timestamp).strftime('%a %b %d %H:%M:%S %Y')
-        
-        num_of_equals = (self.terminal_size - len(formatted_time)) // 2
-        
-        result = '=' * num_of_equals + formatted_time + '=' * num_of_equals
-        print(result)
 
-    def __print_bottom_edge(self):
-        print('-' * self.terminal_size)
+    def print_table_format(self, groupby, columns):
+        table_fields=[]
+        for item in self.objects_matched:
+            is_print_table = True
+            tags   = item["tags"]
+            fields = item["fields"]
+            
+            if groupby not in tags:
+                continue
+            for column in columns:
+                if column not in fields:
+                    is_print_table = False
+                    break
+                if isinstance(fields[column], str):
+                    is_print_table = False
+                    continue
+            if is_print_table == False:
+                continue
+
+            new_fields = copy.deepcopy(item)
+            
+            for key in fields:
+                if key not in columns:
+                    new_fields["fields"].pop(key)
 
-    def print_data(self):
-        self.__print_top_edge()
+            for key in tags:
+                if groupby != key:
+                    new_fields["tags"].pop(key)
 
-        if self.display_counter:
-            self.ctable.print_tables()
+            table_fields.append(new_fields)
+        return self.build_table_format_exporter(table_fields)
+    
+#   def print_counter(self, [tags], [fields])
+    def print_counter_type(self, field_keys):
+        counter_fields = []
+        for item in self.objects_matched:
+            fields = item["fields"]
 
-        if self.display_hist:
-            self.htable.print_tables()
+            new_fields = copy.deepcopy(item)
+                    
+            for key in fields:
+                if (key not in field_keys) or (isinstance(fields[key], str)):
+                    new_fields["fields"].pop(key)
+            counter_fields.append(new_fields)
+        self.build_not_table_format_exporter(counter_fields)
+        self.ctable.build_columns_tables()
+        str_tables = []
+        for item in self.ctable.tables:
+            str_tables.append(item.get_string())        
+        return "".join(str_tables)
+
+    def print_histogram_type(self, field_keys):
+        hist_fields = []
+        for item in self.objects_matched:
+            fields = item["fields"]
+            new_fields = copy.deepcopy(item) 
+            for key in fields:
+                if (key not in field_keys) \
+                    or (not isinstance(fields[key], str)) \
+                    or (True == FieldstatAPI.libfieldstat.fieldstat_is_hll(fields[key].encode('utf-8'))):
+                    new_fields["fields"].pop(key)
+            hist_fields.append(new_fields)
+        self.build_not_table_format_exporter(hist_fields)
+        str_tables = []
+        for item in self.htable.tables:
+            str_tables.append(item.get_string())   
+        return "".join(str_tables)
+
+    def print_hll_type(self, field_keys):
+        hll_fields = []
+        for item in self.objects_matched:
+            fields = item["fields"]
+            new_fields = copy.deepcopy(item) 
+            for key in fields:
+                if (key not in field_keys) \
+                    or (not isinstance(fields[key], str)) \
+                    or (False == FieldstatAPI.libfieldstat.fieldstat_is_hll(fields[key].encode('utf-8'))):
+                    new_fields["fields"].pop(key)
+                    
+            hll_fields.append(new_fields)
+        self.build_not_table_format_exporter(hll_fields)
+        self.hlltable.build_columns_tables()
+        str_tables = []
+        for item in self.hlltable.tables:
+            str_tables.append(item.get_string())        
+        return "".join(str_tables)
+
+
+    def print_local_exporter(self):
+        #The top edge
+        timestamp      = datetime.datetime.now().timestamp()
+        formatted_time = datetime.datetime.fromtimestamp(timestamp).strftime('%a %b %d %H:%M:%S %Y')
+        num_of_equals  = (self.terminal_size - len(formatted_time)) // 2
+        print('=' * num_of_equals + formatted_time + '=' * num_of_equals)
 
-        if self.display_hll:
-            self.hlltable.print_tables()
+        if len(self.template) > 0:
+            print(self.template_ja2.render())
+        else:
+            if self.display_counter:
+                self.ctable.print_tables()
+                print("=" * self.terminal_size)
+                self.tftable.print_tables()
+                print("=" * self.terminal_size)
 
-        self.__print_bottom_edge()
+            if self.display_hist:
+                self.htable.print_tables()
+                print("=" * self.terminal_size)
+
+            if self.display_hll:
+                self.hlltable.print_tables()
+                print("=" * self.terminal_size)
+
+        #The bottom edge
+        print('-' * self.terminal_size)
 
 
     @classmethod
     def run_local_exporter(cls):
-        praser = cls()
-        praser.parse_data()
-        praser.print_data()
-
+        exporter = cls()
+        exporter.build_local_exporter()
+        exporter.print_local_exporter()
 
 
 ################################################################################
@@ -635,10 +776,10 @@ class FieldstatExporter:
                             help = 'Display histogram type metrics')
         parser.add_argument('--display-counter', action = 'store_true', default = False,
                             help = 'Display counter type metrics')
-        parser.add_argument('--disable-table',   action = 'store_true', default = False,
-                            help = 'disable display table format')
         parser.add_argument("-m", "--match-tags", type = str,           default = "",
                             help = "Display the tags match metrics")
+        parser.add_argument("-t", "--template",   type = str,           default = "",
+                            help = "Specify the print template with jinja2.")
 
 
     def __parse_bins_str(self, bins_str):
@@ -682,7 +823,7 @@ class FieldstatExporter:
             FieldstatExporterVars.local_display_hll     = args.display_hll
             FieldstatExporterVars.local_display_hist    = args.display_hist
             FieldstatExporterVars.local_match_tags      = self.__parse_tags_str(args.match_tags)
-            FieldstatExporterVars.local_disable_table   = args.disable_table
+            FieldstatExporterVars.local_template        = args.template
             self.exporter_mode       = 'local'
             self.local_interval_s    = args.interval
             self.local_enable_loop   = args.loop