diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/exporter/fieldstat_exporter.py | 108 |
1 files changed, 42 insertions, 66 deletions
diff --git a/src/exporter/fieldstat_exporter.py b/src/exporter/fieldstat_exporter.py index ac1f9ce..5b80530 100644 --- a/src/exporter/fieldstat_exporter.py +++ b/src/exporter/fieldstat_exporter.py @@ -68,7 +68,7 @@ class FieldstatExporterVars: prom_uri_path = "" - json_path = "" + json_paths = [] hist_format = "" hist_bins = [] @@ -81,7 +81,7 @@ class PrometheusExporter: def __init__(self): self.hist_bins = FieldstatExporterVars.hist_bins self.hist_format = FieldstatExporterVars.hist_format - self.json_path = FieldstatExporterVars.json_path + self.json_paths = FieldstatExporterVars.json_paths self.n_lines = 0 def __escape_metric_name(self, metric_name): @@ -171,14 +171,14 @@ class PrometheusExporter: metrics += self.__build_type_histogram(escaped_name, escaped_tags, value) return metrics - def build_metrics_payload(self): + def build_metrics_payload(self, json_path): payload = "" - if not os.path.exists(self.json_path): - logging.error("Path: {%s} does not exist", self.json_path) + if not os.path.exists(json_path): + logging.error("Path: {%s} does not exist", json_path) return payload - with open(self.json_path) as file: + with open(json_path) as file: fcntl.flock(file, fcntl.LOCK_EX) json_data = [] try: @@ -191,13 +191,19 @@ class PrometheusExporter: return payload + def merge_multi_files_payload(self): + payloads = "" + for item in self.json_paths: + payloads += self.build_metrics_payload(item) + return payloads + def read_lines_num(self): return self.n_lines @classmethod def run_prometheus_exporter(cls): builder = cls() - return builder.build_metrics_payload() + return builder.merge_multi_files_payload() class PrometheusEndpoint(BaseHTTPRequestHandler): def __init__(self, request, client_address, server): @@ -471,11 +477,11 @@ class HistogramTable: class LocalExporter: def __init__(self): self.terminal_size, _ = shutil.get_terminal_size((128, 64)) - self.json_path = FieldstatExporterVars.json_path - self.ctable = CounterTable() - self.htable = HistogramTable() - self.hlltable = CounterTable() - self.tftable = TableFormatTable() + self.json_paths = FieldstatExporterVars.json_paths + self.ctable = None + self.htable = None + self.hlltable = None + self.tftable = None self.display_counter = FieldstatExporterVars.local_display_counter self.display_hist = FieldstatExporterVars.local_display_hist self.display_hll = FieldstatExporterVars.local_display_hll @@ -483,7 +489,7 @@ class LocalExporter: self.template = FieldstatExporterVars.local_template self.__set_default_display() self.objects_matched = [] - self.template_ja2 = None + self.template_ja2 = None def __set_default_display(self): @@ -531,13 +537,13 @@ class LocalExporter: return True - def read_json_objects_from_file(self): + def read_json_objects_from_file(self, json_path): #check source json file is exist. objects = [] - if not os.path.exists(self.json_path): - logging.error("Path: {%s} does not exist", self.json_path) + if not os.path.exists(json_path): + logging.error("Path: {%s} does not exist", json_path) return objects - with open(self.json_path) as fd: + with open(json_path) as fd: fcntl.flock(fd, fcntl.LOCK_EX) try: objects = json.load(fd) @@ -546,7 +552,6 @@ class LocalExporter: fcntl.flock(fd, fcntl.LOCK_UN) return objects - def read_match_tags_json_objects(self, json_objects): matched_objects = [] @@ -566,42 +571,6 @@ class LocalExporter: return matched_objects - # def build_table_format_htable(self, json_objects): - # table_bundle = {} - # table_bundle["not_table_format"] = [] - - # for item in json_objects: - # #set display table option off - # if self.disable_table: - # table_bundle["not_table_format"].append(item) - # continue - # # table: only one tag and same tag key + same field keys + name key - # not_append_table = False - # if len(item["tags"]) != 1: - # table_bundle["not_table_format"].append(item) - # continue - - # for _, value in item["fields"].items(): - # if isinstance(value, str): - # not_append_table = True - # break - - # if not_append_table == True: - # table_bundle["not_table_format"].append(item) - # continue - - # key_list = list(item["tags"].keys()) + sorted(list(item["fields"].keys())) - # key = ''.join(key_list) + item["name"] - - # if key in table_bundle: - # val = table_bundle[key] - # val.append(item) - # else: - # table_bundle[key] = [item] - - # return table_bundle - - def build_not_table_format_exporter(self, json_objects): for item in json_objects: timestamp_ms_delta = item["timestamp_ms_delta"] @@ -647,13 +616,21 @@ class LocalExporter: self.template_ja2 = template def build_local_exporter(self): - objects = self.read_json_objects_from_file() - self.objects_matched = self.read_match_tags_json_objects(objects) - - if len(self.template) > 0: - self.export_templates() - else: - self.build_not_table_format_exporter(self.objects_matched) + for item in self.json_paths: + print("Json file path: " + item) + self.ctable = CounterTable() + self.htable = HistogramTable() + self.hlltable = CounterTable() + self.tftable = TableFormatTable() + self.objects_matched = [] + objects = self.read_json_objects_from_file(item) + self.objects_matched = self.read_match_tags_json_objects(objects) + + if len(self.template) > 0: + self.export_templates() + else: + self.build_not_table_format_exporter(self.objects_matched) + self.print_local_exporter() def print_table_format(self, groupby, columns, enable_speed=True, verbose=False): @@ -788,7 +765,6 @@ class LocalExporter: def run_local_exporter(cls): exporter = cls() exporter.build_local_exporter() - exporter.print_local_exporter() ################################################################################ @@ -798,7 +774,7 @@ class FieldstatExporter: DEFAULT_LISTEN_PORT = 8080 DEFAULT_HIST_BINS = [0.1,0.5,0.8,0.9,0.95,0.99] DEFAULT_HIST_FORMAT = "summary" - DEFAULT_JSON_PATH = "./fieldstat.json" + DEFAULT_JSON_PATHS = ["./fieldstat.json"] DEFAULT_URI_PATH = "/metrics" DEFAULT_INTERVAL_S = 1 @@ -817,8 +793,8 @@ class FieldstatExporter: help = "The metrics of histogram type output bins.") parser.add_argument("-f", "--hist-format", type = str, default = self.DEFAULT_HIST_FORMAT, help = "The metrics of histogram type output format.") - parser.add_argument("-j", "--json-path", type = str, default = self.DEFAULT_JSON_PATH, - help = "The input fieldstat metrics json file path.") + parser.add_argument("-j", "--json-paths", nargs='+', type = str, default = self.DEFAULT_JSON_PATHS, + help = "The input fieldstat metrics json file paths. Support multi path.") return parser def __build_prom_parser(self, subparsers, shared_arg_parser): @@ -876,7 +852,7 @@ class FieldstatExporter: def __read_shared_args_value(self, args): FieldstatExporterVars.hist_format = args.hist_format - FieldstatExporterVars.json_path = args.json_path + FieldstatExporterVars.json_paths = args.json_paths FieldstatExporterVars.hist_bins = self.__parse_bins_str(args.hist_bins) def __read_private_args_value(self, args): |