diff options
| -rw-r--r-- | src/exporter/fieldstat_exporter.py | 108 | ||||
| -rw-r--r-- | test/test_fieldstat_exporter.py | 34 |
2 files changed, 57 insertions, 85 deletions
diff --git a/src/exporter/fieldstat_exporter.py b/src/exporter/fieldstat_exporter.py index ac1f9ce..5b80530 100644 --- a/src/exporter/fieldstat_exporter.py +++ b/src/exporter/fieldstat_exporter.py @@ -68,7 +68,7 @@ class FieldstatExporterVars: prom_uri_path = "" - json_path = "" + json_paths = [] hist_format = "" hist_bins = [] @@ -81,7 +81,7 @@ class PrometheusExporter: def __init__(self): self.hist_bins = FieldstatExporterVars.hist_bins self.hist_format = FieldstatExporterVars.hist_format - self.json_path = FieldstatExporterVars.json_path + self.json_paths = FieldstatExporterVars.json_paths self.n_lines = 0 def __escape_metric_name(self, metric_name): @@ -171,14 +171,14 @@ class PrometheusExporter: metrics += self.__build_type_histogram(escaped_name, escaped_tags, value) return metrics - def build_metrics_payload(self): + def build_metrics_payload(self, json_path): payload = "" - if not os.path.exists(self.json_path): - logging.error("Path: {%s} does not exist", self.json_path) + if not os.path.exists(json_path): + logging.error("Path: {%s} does not exist", json_path) return payload - with open(self.json_path) as file: + with open(json_path) as file: fcntl.flock(file, fcntl.LOCK_EX) json_data = [] try: @@ -191,13 +191,19 @@ class PrometheusExporter: return payload + def merge_multi_files_payload(self): + payloads = "" + for item in self.json_paths: + payloads += self.build_metrics_payload(item) + return payloads + def read_lines_num(self): return self.n_lines @classmethod def run_prometheus_exporter(cls): builder = cls() - return builder.build_metrics_payload() + return builder.merge_multi_files_payload() class PrometheusEndpoint(BaseHTTPRequestHandler): def __init__(self, request, client_address, server): @@ -471,11 +477,11 @@ class HistogramTable: class LocalExporter: def __init__(self): self.terminal_size, _ = shutil.get_terminal_size((128, 64)) - self.json_path = FieldstatExporterVars.json_path - self.ctable = CounterTable() - self.htable = HistogramTable() - self.hlltable = CounterTable() - self.tftable = TableFormatTable() + self.json_paths = FieldstatExporterVars.json_paths + self.ctable = None + self.htable = None + self.hlltable = None + self.tftable = None self.display_counter = FieldstatExporterVars.local_display_counter self.display_hist = FieldstatExporterVars.local_display_hist self.display_hll = FieldstatExporterVars.local_display_hll @@ -483,7 +489,7 @@ class LocalExporter: self.template = FieldstatExporterVars.local_template self.__set_default_display() self.objects_matched = [] - self.template_ja2 = None + self.template_ja2 = None def __set_default_display(self): @@ -531,13 +537,13 @@ class LocalExporter: return True - def read_json_objects_from_file(self): + def read_json_objects_from_file(self, json_path): #check source json file is exist. objects = [] - if not os.path.exists(self.json_path): - logging.error("Path: {%s} does not exist", self.json_path) + if not os.path.exists(json_path): + logging.error("Path: {%s} does not exist", json_path) return objects - with open(self.json_path) as fd: + with open(json_path) as fd: fcntl.flock(fd, fcntl.LOCK_EX) try: objects = json.load(fd) @@ -546,7 +552,6 @@ class LocalExporter: fcntl.flock(fd, fcntl.LOCK_UN) return objects - def read_match_tags_json_objects(self, json_objects): matched_objects = [] @@ -566,42 +571,6 @@ class LocalExporter: return matched_objects - # def build_table_format_htable(self, json_objects): - # table_bundle = {} - # table_bundle["not_table_format"] = [] - - # for item in json_objects: - # #set display table option off - # if self.disable_table: - # table_bundle["not_table_format"].append(item) - # continue - # # table: only one tag and same tag key + same field keys + name key - # not_append_table = False - # if len(item["tags"]) != 1: - # table_bundle["not_table_format"].append(item) - # continue - - # for _, value in item["fields"].items(): - # if isinstance(value, str): - # not_append_table = True - # break - - # if not_append_table == True: - # table_bundle["not_table_format"].append(item) - # continue - - # key_list = list(item["tags"].keys()) + sorted(list(item["fields"].keys())) - # key = ''.join(key_list) + item["name"] - - # if key in table_bundle: - # val = table_bundle[key] - # val.append(item) - # else: - # table_bundle[key] = [item] - - # return table_bundle - - def build_not_table_format_exporter(self, json_objects): for item in json_objects: timestamp_ms_delta = item["timestamp_ms_delta"] @@ -647,13 +616,21 @@ class LocalExporter: self.template_ja2 = template def build_local_exporter(self): - objects = self.read_json_objects_from_file() - self.objects_matched = self.read_match_tags_json_objects(objects) - - if len(self.template) > 0: - self.export_templates() - else: - self.build_not_table_format_exporter(self.objects_matched) + for item in self.json_paths: + print("Json file path: " + item) + self.ctable = CounterTable() + self.htable = HistogramTable() + self.hlltable = CounterTable() + self.tftable = TableFormatTable() + self.objects_matched = [] + objects = self.read_json_objects_from_file(item) + self.objects_matched = self.read_match_tags_json_objects(objects) + + if len(self.template) > 0: + self.export_templates() + else: + self.build_not_table_format_exporter(self.objects_matched) + self.print_local_exporter() def print_table_format(self, groupby, columns, enable_speed=True, verbose=False): @@ -788,7 +765,6 @@ class LocalExporter: def run_local_exporter(cls): exporter = cls() exporter.build_local_exporter() - exporter.print_local_exporter() ################################################################################ @@ -798,7 +774,7 @@ class FieldstatExporter: DEFAULT_LISTEN_PORT = 8080 DEFAULT_HIST_BINS = [0.1,0.5,0.8,0.9,0.95,0.99] DEFAULT_HIST_FORMAT = "summary" - DEFAULT_JSON_PATH = "./fieldstat.json" + DEFAULT_JSON_PATHS = ["./fieldstat.json"] DEFAULT_URI_PATH = "/metrics" DEFAULT_INTERVAL_S = 1 @@ -817,8 +793,8 @@ class FieldstatExporter: help = "The metrics of histogram type output bins.") parser.add_argument("-f", "--hist-format", type = str, default = self.DEFAULT_HIST_FORMAT, help = "The metrics of histogram type output format.") - parser.add_argument("-j", "--json-path", type = str, default = self.DEFAULT_JSON_PATH, - help = "The input fieldstat metrics json file path.") + parser.add_argument("-j", "--json-paths", nargs='+', type = str, default = self.DEFAULT_JSON_PATHS, + help = "The input fieldstat metrics json file paths. Support multi path.") return parser def __build_prom_parser(self, subparsers, shared_arg_parser): @@ -876,7 +852,7 @@ class FieldstatExporter: def __read_shared_args_value(self, args): FieldstatExporterVars.hist_format = args.hist_format - FieldstatExporterVars.json_path = args.json_path + FieldstatExporterVars.json_paths = args.json_paths FieldstatExporterVars.hist_bins = self.__parse_bins_str(args.hist_bins) def __read_private_args_value(self, args): diff --git a/test/test_fieldstat_exporter.py b/test/test_fieldstat_exporter.py index 9106bae..3bbd99c 100644 --- a/test/test_fieldstat_exporter.py +++ b/test/test_fieldstat_exporter.py @@ -221,13 +221,11 @@ class TestPrometheusExporter(unittest.TestCase): def test_build_metrics_payload(self): self.prom.hist_bins = [0.1,0.2,0.5,0.8,0.9,0.95,0.99] self.prom.hist_format = "summary" - self.prom.json_path = "/tmp/invalid_path.json" self.prom.n_lines = 0 - payload = self.prom.build_metrics_payload() + payload = self.prom.build_metrics_payload("/tmp/invalid_path.json") self.assertEqual(payload, "") - self.prom.json_path = FIELDSTAT_INPUT_JSON_PATH - payload = self.prom.build_metrics_payload() + payload = self.prom.build_metrics_payload(FIELDSTAT_INPUT_JSON_PATH) lines = payload.split('\n') self.assertEqual(len(lines), 93 + 1) self.assertEqual(self.prom.read_lines_num(), 93) @@ -381,11 +379,11 @@ class TestTableFormatTable(unittest.TestCase): table_str = table.get_string() row_count = len(table_str.split("\n")) - 1 - self.assertEqual(row_count, 0) + self.assertEqual(row_count, 3) self.c_table.add_table_row(table, tags, field, field_delta, 0, False, columns) row_count = len(table_str.split("\n")) - 1 - self.assertEqual(row_count, 0) + self.assertEqual(row_count, 3) class TestHistogramTable(unittest.TestCase): @@ -567,7 +565,7 @@ class TestLocalExporter(unittest.TestCase): val = random.randint(1, 100) val_delta = random.randint(10, 20) tsms_delta = random.randint(1, 10) * 1000 - self.local.ctable.columns = [] + self.local.ctable = CounterTable() self.local._LocalExporter__build_counter_type_exporter(self.tags, self.key, val, val_delta, tsms_delta) self.assertEqual(self.local.ctable.columns[-1][1][1], str(val)) self.assertEqual(self.local.ctable.columns[-1][1][2], "{:.2f}".format(val_delta*1000/tsms_delta)) @@ -578,7 +576,7 @@ class TestLocalExporter(unittest.TestCase): "AEEAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg"\ "ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgI"\ "CAgICAgICAgICAgICAgICAgICAgIAAAAAAAAA" - + self.local.htable = HistogramTable() peradd = len(self.local.htable.tables) self.local._LocalExporter__build_histogram_type_exporter(self.tags, self.key, hist_val, None, 0) postadd = len(self.local.htable.tables) @@ -587,7 +585,7 @@ class TestLocalExporter(unittest.TestCase): def test__build_hll_type_exporter(self): value = "AQUBDECDAQxAQQUIIEEJCDCFARgQRAUIMIMAAAECAAAAAAAAAA==" - + self.local.hlltable = CounterTable() peradd = len(self.local.hlltable.columns) self.local._LocalExporter__build_hll_type_exporter(self.tags, self.key, value) postadd = len(self.local.hlltable.columns) @@ -668,12 +666,10 @@ class TestLocalExporter(unittest.TestCase): self.local.ctable = CounterTable() self.local.htable = HistogramTable() - self.local.json_path = "/tmp/noexist.json" - objects0 = self.local.read_json_objects_from_file() + objects0 = self.local.read_json_objects_from_file("/tmp/noexist.json") self.assertEqual(len(objects0), 0) - self.local.json_path = FIELDSTAT_INPUT_JSON_PATH - objects1 = self.local.read_json_objects_from_file() + objects1 = self.local.read_json_objects_from_file(FIELDSTAT_INPUT_JSON_PATH) self.assertGreater(len(objects1), 0) @@ -687,11 +683,11 @@ class TestFieldstatExporter(unittest.TestCase): self.assertEqual(parser.parse_args([]).hist_bins, "0.1,0.5,0.8,0.9,0.95,0.99") self.assertEqual(parser.parse_args([]).hist_format, "summary") - self.assertEqual(parser.parse_args([]).json_path, "./fieldstat.json") + self.assertEqual(parser.parse_args([]).json_paths, ["./fieldstat.json"]) self.assertEqual(parser.parse_args(["-b", "0.1,0.5,0.8,0.99"]).hist_bins, "0.1,0.5,0.8,0.99") self.assertEqual(parser.parse_args(["-f", "histogram"]).hist_format, "histogram") - self.assertEqual(parser.parse_args(["-j", "/tmp/sapp_fs.json"]).json_path, "/tmp/sapp_fs.json") + self.assertEqual(parser.parse_args(["-j", "/tmp/sapp_fs.json"]).json_paths, ["/tmp/sapp_fs.json"]) def test_build_prom_parser(self): @@ -750,14 +746,14 @@ class TestFieldstatExporter(unittest.TestCase): self.assertEqual(FieldstatExporterVars.hist_format, "summary") self.assertEqual(FieldstatExporterVars.hist_bins, [0.1, 0.5, 0.8, 0.9, 0.95, 0.99]) - self.assertEqual(FieldstatExporterVars.json_path, "./fieldstat.json") + self.assertEqual(FieldstatExporterVars.json_paths, ["./fieldstat.json"]) args = parser.parse_args(["-f", "histogram", "-b", "1,2,3,4,5", "-j", FIELDSTAT_INPUT_JSON_PATH]) self.exporter._FieldstatExporter__read_shared_args_value(args) self.assertEqual(FieldstatExporterVars.hist_format, "histogram") self.assertEqual(FieldstatExporterVars.hist_bins, [1.0, 2.0, 3.0, 4.0, 5.0]) - self.assertEqual(FieldstatExporterVars.json_path, FIELDSTAT_INPUT_JSON_PATH) + self.assertEqual(FieldstatExporterVars.json_paths, [FIELDSTAT_INPUT_JSON_PATH]) def test_read_private_args_value(self): @@ -839,7 +835,7 @@ class TestFieldstatExporter(unittest.TestCase): self.assertEqual(FieldstatExporterVars.hist_format, "summary") self.assertEqual(FieldstatExporterVars.hist_bins, [0.1, 0.5, 0.8, 0.9, 0.95, 0.99]) - self.assertEqual(FieldstatExporterVars.json_path, "./fieldstat.json") + self.assertEqual(FieldstatExporterVars.json_paths, ["./fieldstat.json"]) self.assertEqual(FieldstatExporterVars.prom_uri_path, "/metrics") self.assertEqual(self.exporter.prom_listen_port, 8080) @@ -851,7 +847,7 @@ class TestFieldstatExporter(unittest.TestCase): self.exporter.read_cmd_options() self.assertEqual(FieldstatExporterVars.hist_format, "summary") self.assertEqual(FieldstatExporterVars.hist_bins, [0.1, 0.5, 0.8, 0.9, 0.95, 0.99]) - self.assertEqual(FieldstatExporterVars.json_path, "./fieldstat.json") + self.assertEqual(FieldstatExporterVars.json_paths, ["./fieldstat.json"]) self.assertEqual(FieldstatExporterVars.local_display_hist, False) self.assertEqual(FieldstatExporterVars.local_display_hll, False) self.assertEqual(FieldstatExporterVars.local_display_counter, False) |