#!/usr/bin/python3 import argparse import sys import json import re import os import logging import shutil import datetime import time import copy import fcntl from itertools import chain from prettytable import PrettyTable,NONE,HEADER from http.server import HTTPServer, BaseHTTPRequestHandler from jinja2 import Environment, FileSystemLoader import ctypes class FieldstatAPI: libfieldstat = ctypes.CDLL('libfieldstat4.so') libfieldstat.fieldstat_histogram_base64_decode.argtypes = [ctypes.c_char_p] libfieldstat.fieldstat_histogram_base64_decode.restype = ctypes.c_void_p libfieldstat.fieldstat_histogram_free.argtypes = [ctypes.c_void_p] libfieldstat.fieldstat_histogram_value_at_percentile_api.argtypes = [ctypes.c_void_p, ctypes.c_double] libfieldstat.fieldstat_histogram_value_at_percentile_api.restype = ctypes.c_longlong libfieldstat.fieldstat_histogram_count_le_value_api.argtypes = [ctypes.c_void_p, ctypes.c_longlong] libfieldstat.fieldstat_histogram_count_le_value_api.restype = ctypes.c_longlong libfieldstat.fieldstat_histogram_value_total_count.argtypes = [ctypes.c_void_p] libfieldstat.fieldstat_histogram_value_total_count.restype = ctypes.c_longlong libfieldstat.fieldstat_histogram_value_min.argtypes = [ctypes.c_void_p] libfieldstat.fieldstat_histogram_value_min.restype = ctypes.c_longlong libfieldstat.fieldstat_histogram_value_max.argtypes = [ctypes.c_void_p] libfieldstat.fieldstat_histogram_value_max.restype = ctypes.c_longlong libfieldstat.fieldstat_histogram_value_mean.argtypes = [ctypes.c_void_p] libfieldstat.fieldstat_histogram_value_mean.restype = ctypes.c_double libfieldstat.fieldstat_histogram_value_stddev.argtypes = [ctypes.c_void_p] libfieldstat.fieldstat_histogram_value_stddev.restype = ctypes.c_double libfieldstat.fieldstat_hll_base64_to_count.argtypes = [ctypes.c_char_p] libfieldstat.fieldstat_hll_base64_to_count.restype = ctypes.c_double libfieldstat.fieldstat_histogram_value_sum.argtypes = [ctypes.c_void_p] libfieldstat.fieldstat_histogram_value_sum.restype = ctypes.c_longlong libfieldstat.fieldstat_is_hll.argtypes = [ctypes.c_char_p] libfieldstat.fieldstat_is_hll.restype = ctypes.c_bool logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(message)s') ################################################################################ # global variables ################################################################################ class FieldstatExporterVars: local_display_counter = False local_display_hist = False local_display_hll = False local_match_tags = {} local_template = "" local_display_notagged = False prom_uri_path = "" json_paths = [] json_dirs = [] hist_format = "" hist_bins = [] local_scientific_form = False ################################################################################ # global functions ################################################################################ def read_json_paths_from_dirs(dirs): json_paths = [] for item in dirs: if not os.path.exists(item): logging.error("Dir {%s} is not exist.", item) continue if not os.path.isdir(item): logging.error("Path {%s} is not directory.", item) continue for file in os.listdir(item): if file.endswith(".json"): file_path = os.path.abspath(os.path.join(item, file)) json_paths.append(file_path) return json_paths ################################################################################ # promethues exporter ################################################################################ class PrometheusExporter: def __init__(self): self.hist_bins = FieldstatExporterVars.hist_bins self.hist_format = FieldstatExporterVars.hist_format self.json_paths = copy.deepcopy(FieldstatExporterVars.json_paths) self.json_paths.extend(read_json_paths_from_dirs(FieldstatExporterVars.json_dirs)) self.n_lines = 0 def __escape_metric_name(self, metric_name): #regex: [a-zA-Z_:][a-zA-Z0-9_:]* name = re.sub(r'[^a-zA-Z0-9_:]', '_', metric_name) # remove the last underscore for i in range(len(name) - 1, -1, -1): if name[i] != '_': break name = name[:i] if name == "": return "_" return name def __escape_metric_tags(self, json_dict): #regex: [a-zA-Z_][a-zA-Z0-9_]* dst_tags = [] src_tags = json_dict["tags"] src_tags["app_name"] = json_dict["name"] for key,value in src_tags.items(): dst_key = re.sub(r'[^a-zA-Z0-9_:]', '_', key) dst_val = str(value) dst_tags.append(f'{dst_key}="{dst_val}"') return ','.join(dst_tags) def __build_type_counter(self, name, tags, value): metric = name + "{" + tags + "}" + ' ' + str(value) + '\n' self.n_lines += 1 return metric def __build_histogram_format(self, name, tags, c_hist): metrics = "" for i in self.hist_bins: value = FieldstatAPI.libfieldstat.fieldstat_histogram_count_le_value_api(c_hist, int(i)) metric = name + "_bucket" + "{" + tags + ",le=\"{:.2f}\"".format(i) + "}" + ' ' + str(value) + '\n' metrics += metric self.n_lines += 1 return metrics def __build_summary_format(self, name, tags, c_hist): metrics = "" for i in self.hist_bins: value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_at_percentile_api(c_hist, float(i * 100)) metric = name + "{" + tags + ",quantile=\"{:.2f}\"".format(i) + "}" + ' ' + str(value) + '\n' metrics += metric self.n_lines += 1 return metrics def __build_type_histogram(self, name, tags, value): metrics = "" c_hist = FieldstatAPI.libfieldstat.fieldstat_histogram_base64_decode(value.encode('utf-8')) if self.hist_format == "summary": metrics += self.__build_summary_format(name, tags, c_hist) if self.hist_format == "histogram": metrics += self.__build_histogram_format(name, tags, c_hist) sum_value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_sum(c_hist) metrics += name + "_sum" + "{" + tags + "}" + ' ' + str(sum_value) + '\n' self.n_lines += 1 cnt_value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_total_count(c_hist) metrics += name + "_count" + "{" + tags + "}" + ' ' + str(cnt_value) + '\n' self.n_lines += 1 FieldstatAPI.libfieldstat.fieldstat_histogram_free(c_hist) return metrics def __build_type_hll(self, name, tags, value): hll_value = FieldstatAPI.libfieldstat.fieldstat_hll_base64_to_count(value.encode('utf-8')) self.n_lines += 1 return name + "{" + tags + "}" + ' ' + "{:.2f}".format(hll_value) + '\n' def __build_metrics(self, json_dict): # convert one cell to payload format metrics = "" escaped_tags = self.__escape_metric_tags(json_dict) used_metrics = {} for key,value in json_dict["fields"].items(): escaped_name = self.__escape_metric_name(key) if escaped_name in used_metrics: used_metrics[escaped_name] += 1 escaped_name = escaped_name + "_" + str(used_metrics[escaped_name]) else: used_metrics[escaped_name] = 0 if isinstance(value, int): metrics += self.__build_type_counter(escaped_name, escaped_tags, value) else: is_hll = FieldstatAPI.libfieldstat.fieldstat_is_hll(value.encode('utf-8')) if is_hll: metrics += self.__build_type_hll(escaped_name, escaped_tags, value) else: metrics += self.__build_type_histogram(escaped_name, escaped_tags, value) return metrics def build_metrics_payload(self, json_path): payload = "" if not os.path.exists(json_path): logging.error("Path: {%s} does not exist", json_path) return payload with open(json_path) as file: fcntl.flock(file, fcntl.LOCK_EX) json_data = [] try: json_data = json.load(file) except json.JSONDecodeError as e: print("Invalid JSON:", e) fcntl.flock(file, fcntl.LOCK_UN) for item in json_data: # the item has __name__ in its global tag, move it to the name if "name" in item["tags"]: item["name"] = item["tags"]["name"] item["tags"].pop("name", None) else: item["name"] = "-" payload += self.__build_metrics(item) return payload def merge_multi_files_payload(self): payloads = "" for item in self.json_paths: payloads += self.build_metrics_payload(item) return payloads def read_lines_num(self): return self.n_lines @classmethod def run_prometheus_exporter(cls): builder = cls() return builder.merge_multi_files_payload() class PrometheusEndpoint(BaseHTTPRequestHandler): def __init__(self, request, client_address, server): self.desired_path = FieldstatExporterVars.prom_uri_path super().__init__(request, client_address, server) def do_GET(self): if self.path == self.desired_path: resp = PrometheusExporter.run_prometheus_exporter() self.send_response(200) self.send_header('Content-type', 'text/plain') self.end_headers() self.wfile.write(resp.encode()) else: resp = "uri path:" + self.desired_path self.send_response(404) self.end_headers() self.wfile.write(resp.encode()) ################################################################################ # local exporter ################################################################################ class CounterTable: INFO_COLUMN_WIDTH = 11 # len(speed/s) + 2 * (" ") + 2 * ("|") COLUMM_PADDING = 3 # 2 *(" ") + "|" def __init__(self): self.terminal_size, _ = shutil.get_terminal_size((128, 64)) self.column_size = self.terminal_size - self.INFO_COLUMN_WIDTH self.info_column = ("", ["tags", "sum", "speed/s"]) self.min_width = self.INFO_COLUMN_WIDTH self.tables = [] self.columns = [] self.field_names = [] def __convert_value_to_str(self, value, speed_s): value_str = "" speed_str = "" if FieldstatExporterVars.local_scientific_form == True: value_str = "{:.2e}".format(value) speed_str = "{:.2e}".format(speed_s) else: value_str = str(value) speed_str = "{:.2f}".format(speed_s) return value_str, speed_str def add_table_column(self, tags, head, value, speed_s): value_str, speed_str = self.__convert_value_to_str(value, speed_s) column = (head, [tags, value_str, speed_str]) self.columns.append(column) self.min_width = max(self.min_width, len(tags), len(head), len(value_str)) def __build_one_table(self, columns_slice): table = PrettyTable() table.vrules = NONE table.hrules = NONE # append info column into table table.add_column(self.info_column[0], self.info_column[1], align = "l" ) for item in columns_slice: # append data column into table table.add_column(item[0], item[1], align="r") # parameters max length table.min_width[item[0]] = self.min_width return table def build_columns_tables(self): # One table print in screen size # One table per screen size n_columns = len(self.columns) if n_columns == 0: return table_size = self.column_size // (self.min_width + self.COLUMM_PADDING) if 0 == table_size: table_size = 1 for i in range(0, n_columns, table_size): table = None l_edge = i r_edge = min(i + table_size, n_columns) if r_edge >= n_columns: table = self.__build_one_table(self.columns[l_edge:]) else: table = self.__build_one_table(self.columns[l_edge:r_edge]) self.tables.append(table) def read_columns_num(self): return len(self.columns) def read_rows_num(self): return len(self.rows) def read_tables_num(self): return len(self.tables) def print_tables(self): self.build_columns_tables() for item in self.tables: print(item) if len(self.tables) > 0: print("=" * self.terminal_size) class HistogramTable: def __init__(self): self.format = FieldstatExporterVars.hist_format self.bins = FieldstatExporterVars.hist_bins self.tables = [] self.terminal_size, _ = shutil.get_terminal_size((128, 64)) def __get_row_shared_values(self, c_hist): shared_values = [] max_value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_max(c_hist) shared_values.append(str(max_value)) min_value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_min(c_hist) shared_values.append(str(min_value)) avg_value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_mean(c_hist) shared_values.append("{:.2f}".format(avg_value)) dev_value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_stddev(c_hist) shared_values.append("{:.2f}".format(dev_value)) cnt_value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_total_count(c_hist) shared_values.append(str(cnt_value)) return shared_values def __get_row_values(self, c_hist): row_values = [] for i in self.bins: if self.format == "summary": value = FieldstatAPI.libfieldstat.fieldstat_histogram_value_at_percentile_api(c_hist, float(i * 100)) row_values.append(str(value)) if self.format == "histogram": value = FieldstatAPI.libfieldstat.fieldstat_histogram_count_le_value_api(c_hist, int(i)) row_values.append(str(value)) shared_values = self.__get_row_shared_values(c_hist) row_values += shared_values return row_values def __add_table_field_names(self, table): field_names = [] field_names.append("") for i in self.bins: if self.format == "summary": field_names.append("{:.2f}%".format(i * 100)) if self.format == "histogram": field_names.append("le={:d}".format(int(i))) field_names += ["MAX", "MIN", "AVG", "STDDEV", "CNT"] table.field_names = field_names for item in field_names: table.min_width[item] = 10 if item == "": table.align[item] = "l" else: table.align[item] = "r" def __add_table_row(self, table, hist_value, row_name): if hist_value is None: return c_hist = FieldstatAPI.libfieldstat.fieldstat_histogram_base64_decode(hist_value.encode('utf-8')) row = [row_name] + self.__get_row_values(c_hist) FieldstatAPI.libfieldstat.fieldstat_histogram_free(c_hist) table.add_row(row) def build_table(self, tags, key, hist_value, hist_value_delta, interval_ms): table = PrettyTable() table.vrules = NONE # table.hrules = NONE table.title = key + " " + tags + " " if interval_ms > 0: table.title += "interval: {:d}ms".format(int(interval_ms)) self.__add_table_field_names(table) self.__add_table_row(table, hist_value, "acc") self.__add_table_row(table, hist_value_delta, "delta") self.tables.append(table) def read_tables_num(self): return len(self.tables) def print_tables(self): for item in self.tables: print(item) if len(self.tables) > 0: print("=" * self.terminal_size) def print_dictionary_with_alignment(dict, terminal_width, padding_grid): current_line_length = 0 for key, value in dict.items(): pair_length = len(key) + len(str(value)) + 1 # estimate the length used by current pair if pair_length > padding_grid: padding = (pair_length // padding_grid + 1) * padding_grid else: padding = padding_grid # if current line space is not enough, then start a new line if current_line_length + padding > terminal_width and current_line_length != 0: print() current_line_length = 0 # print by padding the space until the length of the pair is equal to padding print(f"{key}:{value}", end=' ' * (padding - pair_length)) current_line_length += padding print() print("-" * terminal_width) class hashable_tag: def __init__(self, tags): self.tags = tags def __eq__(self, other): return self.tags == other.tags def __hash__(self): return hash(frozenset(self.tags.items())) class JsonParser: def __init__(self, json_objects): # json_objects is a list of json object {tags, fields, fields_delta, timestamp_ms, timestamp_ms_delta} self.tag_info_map = {} # tag:{fieldname, delta fields, timestamp, timestamp_delta} for item in json_objects: tags = item["tags"] delta_fields = item["fields_delta"] if "fields_delta" in item else {} timestamp_delta = item["timestamp_ms_delta"] if "timestamp_ms_delta" in item else -1 self.tag_info_map[hashable_tag(tags)] = (item["fields"], delta_fields, item["timestamp_ms"], timestamp_delta) def get_v(self, tags, field_name): if hashable_tag(tags) not in self.tag_info_map: return "invalid tag" i_tuple = self.tag_info_map[hashable_tag(tags)] fields = i_tuple[0] if field_name[0] == "%" and field_name[-1] == "%": field_name = field_name[1:-1] scientific_form = True else: scientific_form = False def sci_note(v): if isinstance(v, str): return v if scientific_form: return "{:.2e}".format(v) if isinstance(v, float): return "{:.2f}".format(v) return v pattern = r'#Name<(.*?)>' match = re.search(pattern, field_name) if match: field_name = re.sub(pattern, '', field_name) if field_name in fields: tmp_v = fields[field_name] if isinstance(tmp_v, str): return "field_value_is_string" return sci_note(tmp_v) # try special field name if field_name.startswith("#") == False: return "-" pattern = re.compile(r'#Ratio<(\w+),(\w+)(,.+?)?>') match = pattern.match(field_name) if match: field_n = match.group(1) field_d = match.group(2) scaling = match.group(3) if scaling: scaling = float(scaling[1:]) if field_n in fields and field_d in fields: if isinstance(fields[field_n], str) or isinstance(fields[field_d], str): return "field_value_is_string" if fields[field_d] != 0: tmp_v = fields[field_n] / fields[field_d] else: if fields[field_n] == 0: tmp_v = 0 else: return "-" if scaling: if scaling > 0: tmp_v *= scaling else: tmp_v /= -scaling return sci_note(tmp_v) else: return "-" pattern = re.compile(r'#Speed<(\w+)>') match = pattern.match(field_name) if match: fields_delta = i_tuple[1] timestamp_ms_delta = i_tuple[3] f_name = match.group(1) speed_fields = f_name + "/s" if f_name in fields_delta: tmp_v = fields_delta[f_name] if isinstance(tmp_v, str): return "field_value_is_string" tmp_v = tmp_v * 1000 / timestamp_ms_delta if timestamp_ms_delta > 0 else "-" return sci_note(tmp_v) else: return "-" return "-" def convert_to_header_name(name): if name[0] == "%" and name[-1] == "%": name = name[1:-1] pattern = r'#Name<(.*?)>' match = re.search(pattern, name) if match: return match.group(1) if name.startswith("#"): pattern = re.compile(r'#Ratio<(\w+),(\w+)>') match = pattern.match(name) if match: a = match.group(1) b = match.group(2) return f"{a}/{b}" pattern = re.compile(r'#Speed<(\w+)>') match = pattern.match(name) if match: return f"{match.group(1)}/s" return name class LocalExporter: def __init__(self): self.terminal_size, _ = shutil.get_terminal_size((128, 64)) self.json_paths = copy.deepcopy(FieldstatExporterVars.json_paths) self.json_paths.extend(read_json_paths_from_dirs(FieldstatExporterVars.json_dirs)) self.ctable = None self.htable = None self.hlltable = None self.tftable = None self.display_counter = FieldstatExporterVars.local_display_counter self.display_hist = FieldstatExporterVars.local_display_hist self.display_hll = FieldstatExporterVars.local_display_hll self.match_tags = FieldstatExporterVars.local_match_tags self.template = FieldstatExporterVars.local_template self.display_notagged = FieldstatExporterVars.local_display_notagged self.__set_default_display() self.objects_matched = [] self.template_ja2 = None def __set_default_display(self): if not (self.display_counter or self.display_hist or self.display_hll): self.display_counter = True self.display_hist = True self.display_hll = True def __build_counter_type_exporter(self, tags, key, value, value_delta, timestamp_ms_delta): speed_s = 0.0 if (value_delta is not None) and (timestamp_ms_delta > 0): speed_s = value_delta * 1000 / timestamp_ms_delta self.ctable.add_table_column(tags, key, value, speed_s) def __build_histogram_type_exporter(self, tags, key, value, value_delta, timestamp_ms_delta): self.htable.build_table(tags, key, value, value_delta, timestamp_ms_delta) def __build_hll_type_exporter(self, tags, key, value): hll_value = FieldstatAPI.libfieldstat.fieldstat_hll_base64_to_count(value.encode('utf-8')) self.hlltable.add_table_column(tags, key, "{:.2f}".format(hll_value), 0) def __is_tags_matched(self, tags): if len(self.match_tags) == 0: return True if len(tags) == 0: return False if len(self.match_tags) > len(tags): return False for key, value in self.match_tags.items(): if key not in tags: return False if value != tags[key]: return False return True def read_json_objects_from_file(self, json_path): #check source json file is exist. objects = [] if not os.path.exists(json_path): logging.error("Path: {%s} does not exist", json_path) return objects with open(json_path) as fd: fcntl.flock(fd, fcntl.LOCK_EX) try: objects = json.load(fd) for obj in objects: if "name" in obj["tags"]: obj["name"] = obj["tags"]["name"] obj["tags"].pop("name", None) else: obj["name"] = "-" except json.JSONDecodeError as e: print("Invalid JSON:", e) fcntl.flock(fd, fcntl.LOCK_UN) return objects def read_match_tags_json_objects(self, json_objects): matched_objects = [] for item in json_objects: tags = item["tags"] if len(tags) == 0: continue #not match tags object. not read. if not self.__is_tags_matched(tags): continue #match tags object. delete matching tags. for key,value in self.match_tags.items(): if key in tags and value == tags[key]: tags.pop(key, None) matched_objects.append(item) self.objects_matched = matched_objects return matched_objects def build_not_table_format_exporter(self, json_objects): for item in json_objects: if "timestamp_ms_delta" not in item: timestamp_ms_delta = 0 else: timestamp_ms_delta = item["timestamp_ms_delta"] if "fields_delta" not in item: fields_delta = {} else: fields_delta = item["fields_delta"] tags_new = item["tags"] tags_new = json.dumps(tags_new) for key,value in item["fields"].items(): value_delta = None if key in fields_delta: value_delta = fields_delta[key] if not isinstance(value, str): self.__build_counter_type_exporter(tags_new, key, value, value_delta, timestamp_ms_delta) else: # histogram and hll type is_hll = FieldstatAPI.libfieldstat.fieldstat_is_hll(value.encode('utf-8')) if is_hll: self.__build_hll_type_exporter(tags_new, key, value) else: self.__build_histogram_type_exporter(tags_new, key, value, value_delta, timestamp_ms_delta) def export_templates(self): env = Environment(loader=FileSystemLoader('templates')) env.globals.update(print_tables =self.print_table_format) env.globals.update(print_counters =self.print_counter_type) env.globals.update(print_histograms=self.print_histogram_type) env.globals.update(print_hlls =self.print_hll_type) template = env.from_string(self.template) self.template_ja2 = template def build_local_exporter(self): for item in self.json_paths: print("Json file path: " + item) self.ctable = CounterTable() self.htable = HistogramTable() self.hlltable = CounterTable() self.objects_matched = [] objects = self.read_json_objects_from_file(item) self.objects_matched = self.read_match_tags_json_objects(objects) self.fields_untagged = [item for item in objects if len(item["tags"]) == 0] if len(self.match_tags) == 0 else [] exporter_name = objects[0]["name"] if len(objects) > 0 else None if len(self.template) > 0: self.export_templates() else: self.build_not_table_format_exporter(self.objects_matched + self.fields_untagged) self.print_local_exporter(exporter_name) def print_table_format(self, groupby, columns, verbose=False): parser = JsonParser(self.objects_matched) ret = PrettyTable() ret.vrules = NONE header = [""] header += [convert_to_header_name(x) for x in columns] ret.field_names = header ret.align[""] = "l" for i in header[1:]: ret.align[i] = "r" for item in self.objects_matched: tags = item["tags"] if groupby not in tags: continue if verbose: row_str = [] for key, value in tags.items(): row_str.append("%s_%s" % (key, str(value))) row = ['&'.join(row_str)] else: row = [groupby + "_" + str(tags[groupby])] row += [parser.get_v(tags, x) for x in columns] if FieldstatExporterVars.local_scientific_form: row = ["{:.2e}".format(float(x)) if isinstance(x, int) else x for x in row] ret.add_row(row) return str(ret) + '\n' def print_counter_type(self, field_keys): counter_fields = [] for item in chain(self.objects_matched, self.fields_untagged): fields = item["fields"] new_fields = copy.deepcopy(item) for key in fields: if (key not in field_keys) or (isinstance(fields[key], str)): new_fields["fields"].pop(key) counter_fields.append(new_fields) self.build_not_table_format_exporter(counter_fields) self.ctable.build_columns_tables() str_tables = [] for item in self.ctable.tables: str_tables.append(item.get_string()) return "".join(str_tables) def print_histogram_type(self, field_keys): hist_fields = [] for item in self.objects_matched: fields = item["fields"] new_fields = copy.deepcopy(item) for key in fields: if (key not in field_keys) \ or (not isinstance(fields[key], str)) \ or (True == FieldstatAPI.libfieldstat.fieldstat_is_hll(fields[key].encode('utf-8'))): new_fields["fields"].pop(key) hist_fields.append(new_fields) self.build_not_table_format_exporter(hist_fields) str_tables = [] for item in self.htable.tables: str_tables.append(item.get_string()) return "".join(str_tables) def print_hll_type(self, field_keys): hll_fields = [] for item in self.objects_matched: fields = item["fields"] new_fields = copy.deepcopy(item) for key in fields: if (key not in field_keys) \ or (not isinstance(fields[key], str)) \ or (False == FieldstatAPI.libfieldstat.fieldstat_is_hll(fields[key].encode('utf-8'))): new_fields["fields"].pop(key) hll_fields.append(new_fields) self.build_not_table_format_exporter(hll_fields) self.hlltable.build_columns_tables() str_tables = [] for item in self.hlltable.tables: str_tables.append(item.get_string()) return "".join(str_tables) def print_local_exporter(self, exporter_name): #The top edge timestamp = datetime.datetime.now().timestamp() head_line = datetime.datetime.fromtimestamp(timestamp).strftime('%a %b %d %H:%M:%S %Y') if exporter_name is not None: head_line += " " + exporter_name num_of_equals = (self.terminal_size - len(head_line)) // 2 print('=' * num_of_equals + head_line + '=' * num_of_equals) if self.display_notagged: if len(self.fields_untagged) > 0: print_dictionary_with_alignment(self.fields_untagged[0]["fields"], self.terminal_size, 30) if len(self.template) > 0: print(self.template_ja2.render()) else: if self.display_counter: self.ctable.print_tables() if self.display_hist: self.htable.print_tables() #print("=" * self.terminal_size) if self.display_hll: self.hlltable.print_tables() #print("=" * self.terminal_size) #The bottom edge print('-' * self.terminal_size) @classmethod def run_local_exporter(cls): exporter = cls() exporter.build_local_exporter() def get_jinja_help(args): # 根据参数值打印不同的帮助信息 if args.th == 'print_tables': print("fieldstat_exporter.py local -t '{{ print_tables(, , [verbose mode] }}'") print("- Print a table. Its row is organized by the tag key used to filter the metrics, and its column is organized by the field name list.") print("- Special field name format: #Ratio means the ratio of field1 and field2.\n The third optional parameter scale is to multiply the ratio by a scale factor. If scale is negative, it will divide the ratio by the absolute value of scale. The default scale is 1.\n e.g. #Ratio outputs In bytes/(cputick*1000).") print("- Special field name format: #Speed means printout the speed of the field value changing per second.") print("- Apart from the two format above, add #Name, to change the column name. It's optional, it the column name is not defined, it will use the field name.\n As for ratio, the default name is field1/field2, and for speed, the default name is field/s.") print("- Verbose mode is optional, if true, it will print all the tags used by the shown metrics. The default value is false, and it will only print the tags with keys equal to groupby.") print("- To print out scientific notation for a specific field, use %% in the field name list.\n The percent marks should be used at the outermost. e.g. %#Name#Speed%") print("- Use Example:\n fieldstat_exporter.py local -t '{{ print_tables(\"statistic_items\", [\"T_success_log\",\"T_total_log\",\"%IN_Bytes%\"\"#Name#Ratio\",\"#Speed\"], True) }}'") elif args.th == 'print_counters': print("fieldstat_exporter.py local -t '{{ print_counters()}}'") print("Print the counter type metrics. Only the metrics with name in the field name list will be printed.") elif args.th == 'print_histograms': print("fieldstat_exporter.py local -t '{{ print_histograms()}}'") print("Print the histogram type metrics. Only the metrics with name in the field name list will be printed.") print("the bins and formats of histogram are set by -b and -f separately.") else: print("The -t option only supports 'print_tables', 'print_counters', 'print_histograms'.") print("try -th print_tables, -th print_counters, -th print_histograms for specific usage.") ################################################################################ # fieldstat exporter ################################################################################ class FieldstatExporter: DEFAULT_LISTEN_PORT = 8080 DEFAULT_HIST_BINS = [0.1,0.5,0.8,0.9,0.95,0.99] DEFAULT_HIST_FORMAT = "summary" DEFAULT_URI_PATH = "/metrics" DEFAULT_INTERVAL_S = 1 def __init__(self): self.local_clear_screen = False self.local_enable_loop = False self.local_interval_s = 1 self.prom_listen_port = self.DEFAULT_LISTEN_PORT self.exporter_mode = "local" def __build_shared_args_parser(self): bins_str = ','.join([str(x) for x in self.DEFAULT_HIST_BINS]) parser = argparse.ArgumentParser(add_help=False) parser.add_argument("-b", "--hist-bins", type = str, default = bins_str, help = "The metrics of histogram type output bins.") parser.add_argument("-f", "--hist-format", type = str, default = self.DEFAULT_HIST_FORMAT, help = "The metrics of histogram type output format.") parser.add_argument("-j", "--json-paths", nargs='+', type = str, default = [], help = "The input fieldstat metrics json file paths. Support multi path.") parser.add_argument("-d", "--dirs", nargs='+', type = str, default = [], help ="Specify input fieldstat metrics json file directories.") return parser def __build_prom_parser(self, subparsers, shared_arg_parser): parser = subparsers.add_parser('prometheus', help='Set prometheus exporter', parents=[shared_arg_parser]) parser.add_argument("-p", "--listen-port", type = int, default = self.DEFAULT_LISTEN_PORT, help = "Specify the prometheus endpoint port to listen. i.e., 80,8080. The default is {}.".format(self.DEFAULT_LISTEN_PORT)) parser.add_argument("-u", "--uri-path", type = str, default = self.DEFAULT_URI_PATH, help = "Specify the prometheus endpoint uri path.") def __build_local_parser(self, subparsers, shared_arg_parser): parser = subparsers.add_parser('local', help='Set local exporter', parents=[shared_arg_parser]) parser.add_argument("-i", "--interval", type = int, default = self.DEFAULT_INTERVAL_S, help = "interval, seconds to wait between print.") parser.add_argument("-l", "--loop", action = 'store_true', default = False, help = "print loop, exit when recv a signal.") parser.add_argument('--clear-screen', action = 'store_true', default = False, help = 'clear screen at start of loop') parser.add_argument('--display-hll' , action = 'store_true', default = False, help = 'Display hyperloglog type metrics') parser.add_argument('--display-hist', action = 'store_true', default = False, help = 'Display histogram type metrics') parser.add_argument('--display-counter', action = 'store_true', default = False, help = 'Display counter type metrics') parser.add_argument("-m", "--match-tags", type = str, default = "", help = "Display the tags match metrics") parser.add_argument("-t", "--template", type = str, default = "", help = "Specify the print template with jinja2. For specific usage, try -th") parser.add_argument("-e", "--no-tagged", action = 'store_true', default = False, help = "Print all counter metrics without tags in one list.") parser.add_argument("--scientific-form", action = 'store_true', default = False, help = "Output the counter type value in scientific notation.") parser.add_argument('-th', choices=['print_tables', 'print_counters', 'print_histograms'], help='Print out the help for -t option usage.') def __parse_bins_str(self, bins_str): bins = [] for item in bins_str.split(','): bins.append(float(item)) bins_sort = list(set(bins)) bins_sort.sort() return bins_sort def __parse_tags_str(self, tags_str): tags_dict = {} if not tags_str: return tags_dict pairs = tags_str.split(',') for pair in pairs: key, value = pair.split(':') temp_value = value.strip() if temp_value.isdigit(): tags_dict[key] = int(temp_value) elif re.match(r'^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$', temp_value): tags_dict[key] = float(temp_value) else: tags_dict[key] = temp_value return tags_dict def __read_shared_args_value(self, args): FieldstatExporterVars.hist_format = args.hist_format FieldstatExporterVars.json_paths = args.json_paths FieldstatExporterVars.json_dirs = args.dirs FieldstatExporterVars.hist_bins = self.__parse_bins_str(args.hist_bins) def __read_private_args_value(self, args): if args.command == 'prometheus': FieldstatExporterVars.prom_uri_path = args.uri_path self.exporter_mode = 'prometheus' self.prom_listen_port = args.listen_port if args.command == 'local': FieldstatExporterVars.local_display_counter = args.display_counter FieldstatExporterVars.local_display_hll = args.display_hll FieldstatExporterVars.local_display_hist = args.display_hist FieldstatExporterVars.local_match_tags = self.__parse_tags_str(args.match_tags) FieldstatExporterVars.local_template = args.template FieldstatExporterVars.local_scientific_form = args.scientific_form FieldstatExporterVars.local_display_notagged = args.no_tagged self.exporter_mode = 'local' self.local_interval_s = args.interval self.local_enable_loop = args.loop self.local_clear_screen = args.clear_screen def __verify_cmd_args(self, args): if args.hist_format not in ["summary", "histogram"]: logging.error("Historgram format must be 'summary' or 'histogram'") return -1 bins = args.hist_bins.split(',') if args.hist_format == "summary": for bin in bins: if not re.match(r'^0\.([1-9]|[0][1-9]|[1-9]\d)$', bin): logging.error("When historgram format is %s, bins value in [0.01-0.99], " "bins format example: 0.1,0,2,0.3", args.hist_format) return -1 if args.hist_format == "histogram": for bin in bins: if not re.match(r'^\d+$', bin): logging.error("When historgram format is %s, bins value is integer, " "bins format example: 1,10,100,1000", args.hist_format) return -1 if args.command == 'prometheus': if args.listen_port < 1024 or args.listen_port > 65535: logging.error("invalid listen port, listen port must be in [1024, 65535]") return -1 if args.command == 'local': if args.match_tags != "": pattern = r'^([A-Za-z0-9_-]+:[A-Za-z0-9_\-\.]+,)*([A-Za-z0-9_-]+:[A-Za-z0-9_\-\.]+)$' match = re.match(pattern, args.match_tags) if not match: logging.error("invalid match tags, Example: key1:value1,key2:value2,key3:value3") return -1 return 0 def read_cmd_options(self): parser = argparse.ArgumentParser(description='Fieldstat exporter') shared_args_parser = self.__build_shared_args_parser() subparsers = parser.add_subparsers(dest='command') subparsers.required = True self.__build_prom_parser(subparsers, shared_args_parser) self.__build_local_parser(subparsers, shared_args_parser) args = parser.parse_args() if -1 == self.__verify_cmd_args(args): parser.print_help() sys.exit(1) if args.command == 'local' and args.th: get_jinja_help(args) sys.exit(0) self.__read_shared_args_value(args) self.__read_private_args_value(args) def __enable_prometheus_endpoint(self): server_address = ('', self.prom_listen_port) httpd = HTTPServer(server_address, PrometheusEndpoint) print("Starting Prometheus exporter on {}:{}...".format(FieldstatExporterVars.prom_uri_path, self.prom_listen_port)) httpd.serve_forever() def __enable_local_exporter(self): while True: if self.local_clear_screen: os.system('clear') LocalExporter.run_local_exporter() if not self.local_enable_loop: break; time.sleep(self.local_interval_s) def fieldstat_export(self): if (len(FieldstatExporterVars.json_paths) == 0) and \ (len(FieldstatExporterVars.json_dirs) == 0): logging.error("There is no JSON file path and dir specified. The process will exit.") exit(1) try: if self.exporter_mode == 'prometheus': self.__enable_prometheus_endpoint() if self.exporter_mode == 'local': self.__enable_local_exporter() except KeyboardInterrupt: pass @classmethod def run_fieldstat_exporter(cls): exporter = cls() exporter.read_cmd_options() exporter.fieldstat_export() # -tl # 展示配置 # -tc 创建一个表格列模板 # -tt 创建一个表格模板 # -tc field_name_list # -tt print_tables("tag", ["field1", "field2", "field3"]或者一串数字，表示列模板。) # -t "print_tables('tag', ['field1', 'field2', 'field3'])" 直接手动输入，或者-t 指定表格模板名，或者-t print_tables('tag', 列模板名) # 以下三个都是合法的： # -t "{{ print_tables('tag', ['field1', 'field2', 'field3']) }}{{ print_tables('tag2', ['field1', 'field4']) }}" # -tt "print_tables('tag', ['field1', 'field2', 'field3'])" # -tt “print_tables('tag2', ['field1', 'field4'])” # -t "1 2" # -tc field1 field2 field3 # -t "{{print_tables('tag', 3)}} 2" if __name__ == '__main__': FieldstatExporter.run_fieldstat_exporter()