diff options
| -rw-r--r-- | .gitlab-ci.yml | 1 | ||||
| -rw-r--r-- | CMakeLists.txt | 4 | ||||
| -rw-r--r-- | ci/travis.sh | 3 | ||||
| -rw-r--r-- | ctest/CMakeLists.txt | 4 | ||||
| -rw-r--r-- | readme.md | 134 | ||||
| -rw-r--r-- | test/CMakeLists.txt | 2 | ||||
| -rw-r--r-- | test/test_fieldstat_exporter.py | 6 | ||||
| -rw-r--r-- | test/test_write_json_file.cpp (renamed from test/test_exporter_python.cpp) | 17 |
8 files changed, 166 insertions, 5 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3985e92..64e948d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,6 +19,7 @@ stages: - yum install -y elfutils-libelf-devel - yum install -y libuuid libuuid-devel - yum install -y zlib + - python3 -m pip install prettytable .build_by_travis_for_centos7: diff --git a/CMakeLists.txt b/CMakeLists.txt index 5e1e4d0..65f94ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -137,6 +137,10 @@ set_target_properties(${lib_name}_static PROPERTIES OUTPUT_NAME ${lib_name}) install(TARGETS ${lib_name}_shared LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib COMPONENT LIBRARIES)
install(TARGETS ${lib_name}_static LIBRARY ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib COMPONENT LIBRARIES)
+install(FILES ${PROJECT_SOURCE_DIR}/src/exporter/fieldstat_exporter.py
+ DESTINATION ${CMAKE_INSTALL_PREFIX}/bin
+ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE
+ COMPONENT LIBRARIES)
install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_PREFIX}/include COMPONENT HEADER)
include(Package)
\ No newline at end of file diff --git a/ci/travis.sh b/ci/travis.sh index dc6c83c..bcbd1c2 100644 --- a/ci/travis.sh +++ b/ci/travis.sh @@ -56,6 +56,9 @@ make if [ -n "${UNIT_TEST}" ]; then + current_path=$(pwd) + echo $current_path > /etc/ld.so.conf.d/test_case.conf + ldconfig ctest --verbose fi diff --git a/ctest/CMakeLists.txt b/ctest/CMakeLists.txt index f7d725a..19f4a25 100644 --- a/ctest/CMakeLists.txt +++ b/ctest/CMakeLists.txt @@ -15,6 +15,8 @@ add_test(NAME COPY_GTEST_METRIC_HLL_BINARY COMMAND sh -c "cp ${CMAKE_BINARY_DIR} add_test(NAME COPY_GTEST_REGISTER_AND_RESET_BINARY COMMAND sh -c "cp ${CMAKE_BINARY_DIR}/test/test_register_and_reset ${CMAKE_BINARY_DIR}/testing/") add_test(NAME COPY_GTEST_SERIALIZE_BINARY COMMAND sh -c "cp ${CMAKE_BINARY_DIR}/test/test_serialize ${CMAKE_BINARY_DIR}/testing/") add_test(NAME COPY_GTEST_UNIT_TEST_CELL_MANAGER_BINARY COMMAND sh -c "cp ${CMAKE_BINARY_DIR}/test/unit_test_cell_manager ${CMAKE_BINARY_DIR}/testing/") +add_test(NAME COPY_GTEST_WRITE_JSON_FILE_BINARY COMMAND sh -c "cp ${CMAKE_BINARY_DIR}/test/test_write_json_file ${CMAKE_BINARY_DIR}/testing/") +add_test(NAME CHMOD_UNITTEST COMMAND sh -c "chmod 0755 ${CMAKE_SOURCE_DIR}/test/test_fieldstat_exporter.py; cp ${CMAKE_SOURCE_DIR}/test/test_fieldstat_exporter.py ${CMAKE_BINARY_DIR}/testing/test_fieldstat_exporter") set(GTEST_RUN_DIR ${CMAKE_BINARY_DIR}/testing) add_test(NAME GTEST_EMPTY_TAGS COMMAND test_empty_tags WORKING_DIRECTORY ${GTEST_RUN_DIR}) @@ -27,3 +29,5 @@ add_test(NAME GTEST_METRIC_HLL COMMAND test_metric_hll WORKING_DIRECTORY ${GTEST add_test(NAME GTEST_REGISTER_AND_RESET COMMAND test_register_and_reset WORKING_DIRECTORY ${GTEST_RUN_DIR}) add_test(NAME GTEST_SERIALIZE COMMAND test_serialize WORKING_DIRECTORY ${GTEST_RUN_DIR}) add_test(NAME GTEST_UNIT_TEST_CELL_MANAGER COMMAND unit_test_cell_manager WORKING_DIRECTORY ${GTEST_RUN_DIR}) +add_test(NAME GTEST_WRITE_JSON_FILE COMMAND test_write_json_file WORKING_DIRECTORY ${GTEST_RUN_DIR}) +add_test(NAME GTEST_PYTHON_EXPORTER COMMAND test_fieldstat_exporter WORKING_DIRECTORY ${GTEST_RUN_DIR}) @@ -192,4 +192,136 @@ The same as "copy 1 cell of a counter metric on topk cube" but merge twice. Merging Heavy Keeper is slower the coping it, it is why the execution time is unusually higher than copying it. #### export a cell -I add a instance with 100 cubes, on each of which I register 10 metrics. The max cell number of each cube is 1000, and call the metric operation on every added cell. So there are 100 * 10 * 1000 cells in total. After exporting the instance, calculate the average time spent on each cells.
\ No newline at end of file +I add a instance with 100 cubes, on each of which I register 10 metrics. The max cell number of each cube is 1000, and call the metric operation on every added cell. So there are 100 * 10 * 1000 cells in total. After exporting the instance, calculate the average time spent on each cells. + +## Fieldstat exporter +Fieldstat exporter provides exporter functions for fieldstat4, including prometheus exporter and local exporter. + +### prometheus exporter +The prometheus exporter reads the fieldstat4 json file and enables the prometheus endpoint. The default port is 8080. + +### local exporter +The local exporter reads the fieldstat4 json file and outputs it to the screen. + +### Installation and Usage +Download fieldstat4 rpm from https://repo.geedge.net/pulp/content/ and install rpm package. + + +#### List of rpm files + ... + |-- bin + |-- fieldstat_exporter + |-- lib + |-- ibfieldstat4.a + |-- libfieldstat4.so +#### Service configuration +The following is the prometheus exporter systemd service example file. +```systemd +[Unit] +Description=fieldstat4 prometheus service. +After=target.service #Replace target service with the actual service used. +Requires=target.service #Replace target service with the actual service used. + +[Service] +Type=simple +ExecStart=/opt/tsg/framework/bin/fieldstat_exporter prometheus +RestartSec=10s +Restart=always +PrivateTmp=True + +[Install] +WantedBy=multi-user.target +``` +### Command Line +Fieldstat exporter includes prometheus including prometheus and local subcommands. +```txt +[root@localhost bin]# ./fieldstat_exporter --help +usage: fieldstat_exporter [-h] {prometheus,local} ... + +Fieldstat exporter + +positional arguments: + {prometheus,local} + prometheus Set prometheus exporter + local Set local exporter + +optional arguments: + -h, --help show this help message and exit +``` +The following is prometheus exporter command line example. +```bash +./fieldstat_exporter prometheus -b 0.1,0.2,0.3,0.4,0.5 -f summary -j /tmp/fieldstat.json -p 8080 -u /metrics +``` +The following is fieldstat prometheus exporter help info. +```txt +[root@localhost bin]# ./fieldstat_exporter prometheus --help +usage: fieldstat_exporter prometheus [-h] [-b HIST_BINS] [-f HIST_FORMAT] + [-j JSON_PATH] [-p LISTEN_PORT] + [-u URI_PATH] + +optional arguments: + -h, --help show this help message and exit + -b HIST_BINS, --hist-bins HIST_BINS + The metrics of histogram type output bins. + -f HIST_FORMAT, --hist-format HIST_FORMAT + The metrics of histogram type output format. + -j JSON_PATH, --json-path JSON_PATH + The input fieldstat metrics json file path. + -p LISTEN_PORT, --listen-port LISTEN_PORT + Specify the prometheus endpoint port to listen. i.e., + 80,8080 + -u URI_PATH, --uri-path URI_PATH + Specify the prometheus endpoint uri path +``` +The prometheus exporter optional arguments default values. +args|default value +--- | --- +-b, --hist-bins|[0.1,0.5,0.8,0.9,0.95,0.99] +-f, --hist-format|summary +-j, --json-path|./fieldstat.json +-p, --listen-port|8080 +-u, --uri-path|/metrics +The following is local exporter command line example. +```bash +./fieldstat_exporter local -b 0.1,0.2,0.3,0.4,0.5 -f summary -j /tmp/fieldstat.json -i 1 -l --clear-screen -m policy_id:1,device_name:xxg +``` +The following is fieldstat local exporter help info. +```txt +[root@localhost bin]# ./fieldstat_exporter local --help +usage: fieldstat_exporter local [-h] [-b HIST_BINS] [-f HIST_FORMAT] + [-j JSON_PATH] [-i INTERVAL] [-l] + [--clear-screen] [--display-hll] + [--display-hist] [--display-counter] + [-m MATCH_TAGS] + +optional arguments: + -h, --help show this help message and exit + -b HIST_BINS, --hist-bins HIST_BINS + The metrics of histogram type output bins. + -f HIST_FORMAT, --hist-format HIST_FORMAT + The metrics of histogram type output format. + -j JSON_PATH, --json-path JSON_PATH + The input fieldstat metrics json file path. + -i INTERVAL, --interval INTERVAL + interval, seconds to wait between print. + -l, --loop print loop, exit when recv a signal. + --clear-screen clear screen at start of loop + --display-hll Display hyperloglog type metrics + --display-hist Display histogram type metrics + --display-counter Display counter type metrics + -m MATCH_TAGS, --match-tags MATCH_TAGS + Display the tags match metrics +``` +The local exporter optional arguments default values. +args|default value +--- | --- +-b, --hist-bins|[0.1,0.5,0.8,0.9,0.95,0.99] +-f, --hist-format|summary +-j, --json-path|./fieldstat.json +-i, --interval|1(s) +-l, --loop|False +--clear-screen|False +--display-hll|False +--display-hist|False +--display-counter|False +-m, --match-tags|"" diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3252602..d4cb35a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -57,4 +57,4 @@ add_unit_test(test_performance) add_unit_test(test_register_and_reset)
add_unit_test(test_serialize)
add_unit_test(unit_test_cell_manager)
-add_unit_test(test_exporter_python)
\ No newline at end of file +add_unit_test(test_write_json_file)
\ No newline at end of file diff --git a/test/test_fieldstat_exporter.py b/test/test_fieldstat_exporter.py index 2d2cb22..3a68512 100644 --- a/test/test_fieldstat_exporter.py +++ b/test/test_fieldstat_exporter.py @@ -1,3 +1,5 @@ +#!/usr/bin/python3 + import unittest import sys import urllib @@ -13,8 +15,8 @@ from contextlib import redirect_stdout from prettytable import PrettyTable,NONE,HEADER import os -current_path = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(current_path + '../src/exporter') +current_path = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(current_path + '/../../src/exporter') from fieldstat_exporter import FieldstatAPI diff --git a/test/test_exporter_python.cpp b/test/test_write_json_file.cpp index c8912d7..56bdd73 100644 --- a/test/test_exporter_python.cpp +++ b/test/test_write_json_file.cpp @@ -135,6 +135,19 @@ static struct fieldstat *get_table_fieldstat() return instance; } +static int write_json_to_file(const char *filename, char *json_str) +{ + FILE *fp = fopen(filename, "w"); + if (fp == NULL) + { + return -1; + } + fprintf(fp, json_str); + fclose(fp); + + return 0; +} + TEST(ExporterLocal, TableBuild) { struct timeval current = {100, 10000}; @@ -149,7 +162,9 @@ TEST(ExporterLocal, TableBuild) struct fieldstat_json_exporter *exporter = fieldstat_json_exporter_new(merger); fieldstat_json_exporter_enable_delta(exporter); char *str_json = fieldstat_json_exporter_export(exporter, ¤t); - printf(str_json); + + int ret = write_json_to_file("/tmp/fieldstat.json",str_json); + EXPECT_EQ(0, ret); } |