feature:python exporter add new arg --dirs.v4.4.6

3 files changed, 77 insertions, 27 deletions

diff --git a/readme_fieldstat_easy.md b/readme_fieldstat_easy.md
index 6e39351..67e6901 100644
--- a/readme_fieldstat_easy.md
+++ b/readme_fieldstat_easy.md
@@ -58,9 +58,10 @@ The following is prometheus exporter command line example.
 The following is fieldstat prometheus exporter help info.
 ```txt
 [root@localhost bin]# ./fieldstat_exporter prometheus --help
-usage: fieldstat_exporter prometheus [-h] [-b HIST_BINS] [-f HIST_FORMAT]
-                                     [-j JSON_PATH] [-p LISTEN_PORT]
-                                     [-u URI_PATH]
+usage: fieldstat_exporter.py prometheus [-h] [-b HIST_BINS] [-f HIST_FORMAT]
+                                        [-j JSON_PATHS [JSON_PATHS ...]]
+                                        [-d DIRS [DIRS ...]] [-p LISTEN_PORT]
+                                        [-u URI_PATH]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -68,13 +69,16 @@ optional arguments:
                         The metrics of histogram type output bins.
   -f HIST_FORMAT, --hist-format HIST_FORMAT
                         The metrics of histogram type output format.
-  -j JSON_PATH, --json-path JSON_PATH
-                        The input fieldstat metrics json file path.
+  -j JSON_PATHS [JSON_PATHS ...], --json-paths JSON_PATHS [JSON_PATHS ...]
+                        The input fieldstat metrics json file paths. Support
+                        multi path.
+  -d DIRS [DIRS ...], --dirs DIRS [DIRS ...]
+                        Specify input fieldstat metrics json file directories.
   -p LISTEN_PORT, --listen-port LISTEN_PORT
                         Specify the prometheus endpoint port to listen. i.e.,
-                        80,8080
+                        80,8080. The default is 8080.
   -u URI_PATH, --uri-path URI_PATH
-                        Specify the prometheus endpoint uri path
+                        Specify the prometheus endpoint uri path.
 ```
 The prometheus exporter optional arguments default values.
 args|default value
@@ -91,11 +95,13 @@ The following is local exporter command line example.
 The following is fieldstat local exporter help info.
 ```txt
 [root@localhost bin]# ./fieldstat_exporter local --help
-usage: fieldstat_exporter local [-h] [-b HIST_BINS] [-f HIST_FORMAT]
-                                [-j JSON_PATH] [-i INTERVAL] [-l]
-                                [--clear-screen] [--display-hll]
-                                [--display-hist] [--display-counter]
-                                [-m MATCH_TAGS]
+usage: fieldstat_exporter.py local [-h] [-b HIST_BINS] [-f HIST_FORMAT]
+                                   [-j JSON_PATHS [JSON_PATHS ...]]
+                                   [-d DIRS [DIRS ...]] [-i INTERVAL] [-l]
+                                   [--clear-screen] [--display-hll]
+                                   [--display-hist] [--display-counter]
+                                   [-m MATCH_TAGS] [-t TEMPLATE]
+                                   [--scientific-form]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -103,21 +109,23 @@ optional arguments:
                         The metrics of histogram type output bins.
   -f HIST_FORMAT, --hist-format HIST_FORMAT
                         The metrics of histogram type output format.
-  -j JSON_PATH, --json-path JSON_PATH
-                        The input fieldstat metrics json file path.
+  -j JSON_PATHS [JSON_PATHS ...], --json-paths JSON_PATHS [JSON_PATHS ...]
+                        The input fieldstat metrics json file paths. Support
+                        multi path.
+  -d DIRS [DIRS ...], --dirs DIRS [DIRS ...]
+                        Specify input fieldstat metrics json file directories.
   -i INTERVAL, --interval INTERVAL
                         interval, seconds to wait between print.
   -l, --loop            print loop, exit when recv a signal.
-  --clear-screen        clear screen at start of loop.
-  --display-hll         Display hyperloglog type metrics.
-  --display-hist        Display histogram type metrics.
-  --display-counter     Display counter type metrics.
-  --disable-table       disable display table format.
+  --clear-screen        clear screen at start of loop
+  --display-hll         Display hyperloglog type metrics
+  --display-hist        Display histogram type metrics
+  --display-counter     Display counter type metrics
   -m MATCH_TAGS, --match-tags MATCH_TAGS
                         Display the tags match metrics
   -t TEMPLATE, --template TEMPLATE
                         Specify the print template with jinja2.
-  --scientific-form     Output the value in scientific notation.
+  --scientific-form     Output the counter type value in scientific notation.
 ```
 The local exporter optional arguments default values.
 args|default value
diff --git a/src/exporter/fieldstat_exporter.py b/src/exporter/fieldstat_exporter.py
index 5b80530..0858eaa 100644
--- a/src/exporter/fieldstat_exporter.py
+++ b/src/exporter/fieldstat_exporter.py
@@ -69,12 +69,31 @@ class FieldstatExporterVars:
     prom_uri_path         = ""
 
     json_paths            = []
+    json_dirs             = []
     hist_format           = ""
     hist_bins             = []
 
     local_scientific_form = False
 
 ################################################################################
+#  global functions
+################################################################################
+def read_json_paths_from_dirs(dirs):
+    json_paths = []
+    for item in dirs:
+        if not os.path.exists(item):
+            logging.error("Dir {%s} is not exist.", item)
+            continue
+        if not os.path.isdir(item):
+            logging.error("Path {%s} is not directory.", item)
+            continue
+        for file in os.listdir(item):
+            if file.endswith(".json"):
+                file_path = os.path.abspath(os.path.join(item, file))
+                json_paths.append(file_path)
+    return json_paths
+
+################################################################################
 #  promethues exporter
 ################################################################################
 class PrometheusExporter:
@@ -82,6 +101,7 @@ class PrometheusExporter:
         self.hist_bins   = FieldstatExporterVars.hist_bins
         self.hist_format = FieldstatExporterVars.hist_format
         self.json_paths  = FieldstatExporterVars.json_paths
+        self.json_paths.extend(read_json_paths_from_dirs(FieldstatExporterVars.json_dirs))
         self.n_lines     = 0
 
     def __escape_metric_name(self, metric_name):
@@ -478,6 +498,7 @@ class LocalExporter:
     def __init__(self):
         self.terminal_size, _  = shutil.get_terminal_size((128, 64))
         self.json_paths        = FieldstatExporterVars.json_paths
+        self.json_paths.extend(read_json_paths_from_dirs(FieldstatExporterVars.json_dirs))
         self.ctable            = None
         self.htable            = None
         self.hlltable          = None
@@ -774,7 +795,6 @@ class FieldstatExporter:
     DEFAULT_LISTEN_PORT = 8080
     DEFAULT_HIST_BINS   = [0.1,0.5,0.8,0.9,0.95,0.99]
     DEFAULT_HIST_FORMAT = "summary"
-    DEFAULT_JSON_PATHS  = ["./fieldstat.json"]
     DEFAULT_URI_PATH    = "/metrics"
     DEFAULT_INTERVAL_S  = 1
 
@@ -793,14 +813,16 @@ class FieldstatExporter:
                             help = "The metrics of histogram type output bins.")
         parser.add_argument("-f", "--hist-format", type = str, default = self.DEFAULT_HIST_FORMAT,
                             help = "The metrics of histogram type output format.")
-        parser.add_argument("-j", "--json-paths",  nargs='+',  type = str, default = self.DEFAULT_JSON_PATHS,
+        parser.add_argument("-j", "--json-paths", nargs='+', type = str, default = [],
                             help = "The input fieldstat metrics json file paths. Support multi path.")
+        parser.add_argument("-d", "--dirs",  nargs='+', type = str, default = [],
+                            help ="Specify input fieldstat metrics json file directories.")
         return parser
 
     def __build_prom_parser(self, subparsers, shared_arg_parser):
         parser = subparsers.add_parser('prometheus', help='Set prometheus exporter', parents=[shared_arg_parser])
         parser.add_argument("-p", "--listen-port", type = int, default = self.DEFAULT_LISTEN_PORT,
-                            help = "Specify the prometheus endpoint port to listen. i.e., 80,8080")
+                            help = "Specify the prometheus endpoint port to listen. i.e., 80,8080. The default is {}.".format(self.DEFAULT_LISTEN_PORT))
         parser.add_argument("-u", "--uri-path",    type = str, default = self.DEFAULT_URI_PATH,
                             help = "Specify the prometheus endpoint uri path.")
 
@@ -850,9 +872,25 @@ class FieldstatExporter:
             
         return tags_dict
 
+    def __read_json_paths_from_dirs(self, dirs):
+        json_paths = []
+        for item in dirs:
+            if not os.path.exists(item):
+                logging.error("Dir {%s} is not exist.", item)
+                continue
+            if not os.path.isdir(item):
+                logging.error("Path {%s} is not directory.", item)
+                continue
+            for file in os.listdir(item):
+                if file.endswith(".json"):
+                    file_path = os.path.abspath(os.path.join(item, file))
+                    json_paths.append(file_path)
+        return json_paths
+
     def __read_shared_args_value(self, args):
         FieldstatExporterVars.hist_format = args.hist_format
         FieldstatExporterVars.json_paths  = args.json_paths
+        FieldstatExporterVars.json_dirs   = args.dirs
         FieldstatExporterVars.hist_bins   = self.__parse_bins_str(args.hist_bins)
 
     def __read_private_args_value(self, args):
@@ -941,6 +979,10 @@ class FieldstatExporter:
             time.sleep(self.local_interval_s)
 
     def fieldstat_export(self):
+        if (len(FieldstatExporterVars.json_paths) == 0) and \
+           (len(FieldstatExporterVars.json_dirs)  == 0):
+            logging.error("There is no JSON file path and dir specified. The process will exit.")
+            exit(1)
         try:
             if self.exporter_mode == 'prometheus':
                 self.__enable_prometheus_endpoint()
diff --git a/test/test_fieldstat_exporter.py b/test/test_fieldstat_exporter.py
index 3bbd99c..cf8eb5d 100644
--- a/test/test_fieldstat_exporter.py
+++ b/test/test_fieldstat_exporter.py
@@ -683,7 +683,7 @@ class TestFieldstatExporter(unittest.TestCase):
 
         self.assertEqual(parser.parse_args([]).hist_bins,   "0.1,0.5,0.8,0.9,0.95,0.99")
         self.assertEqual(parser.parse_args([]).hist_format, "summary")
-        self.assertEqual(parser.parse_args([]).json_paths,  ["./fieldstat.json"])
+        self.assertEqual(parser.parse_args([]).json_paths,  [])
 
         self.assertEqual(parser.parse_args(["-b", "0.1,0.5,0.8,0.99"]).hist_bins, "0.1,0.5,0.8,0.99")
         self.assertEqual(parser.parse_args(["-f", "histogram"]).hist_format, "histogram")
@@ -746,7 +746,7 @@ class TestFieldstatExporter(unittest.TestCase):
 
         self.assertEqual(FieldstatExporterVars.hist_format, "summary")
         self.assertEqual(FieldstatExporterVars.hist_bins,   [0.1, 0.5, 0.8, 0.9, 0.95, 0.99])
-        self.assertEqual(FieldstatExporterVars.json_paths,  ["./fieldstat.json"])
+        self.assertEqual(FieldstatExporterVars.json_paths,  [])
 
         args = parser.parse_args(["-f", "histogram", "-b", "1,2,3,4,5", "-j", FIELDSTAT_INPUT_JSON_PATH])
         self.exporter._FieldstatExporter__read_shared_args_value(args)
@@ -835,7 +835,7 @@ class TestFieldstatExporter(unittest.TestCase):
 
         self.assertEqual(FieldstatExporterVars.hist_format,   "summary")
         self.assertEqual(FieldstatExporterVars.hist_bins,     [0.1, 0.5, 0.8, 0.9, 0.95, 0.99])
-        self.assertEqual(FieldstatExporterVars.json_paths,    ["./fieldstat.json"])
+        self.assertEqual(FieldstatExporterVars.json_paths,    [])
 
         self.assertEqual(FieldstatExporterVars.prom_uri_path, "/metrics")
         self.assertEqual(self.exporter.prom_listen_port,      8080)
@@ -847,7 +847,7 @@ class TestFieldstatExporter(unittest.TestCase):
         self.exporter.read_cmd_options()
         self.assertEqual(FieldstatExporterVars.hist_format,           "summary")
         self.assertEqual(FieldstatExporterVars.hist_bins,             [0.1, 0.5, 0.8, 0.9, 0.95, 0.99])
-        self.assertEqual(FieldstatExporterVars.json_paths,            ["./fieldstat.json"])
+        self.assertEqual(FieldstatExporterVars.json_paths,            [])
         self.assertEqual(FieldstatExporterVars.local_display_hist,    False)
         self.assertEqual(FieldstatExporterVars.local_display_hll,     False)
         self.assertEqual(FieldstatExporterVars.local_display_counter, False)