增加APM应用状态监控日志发送功能，向telegraf写入网卡流量、应用处理流量等信息。v4.2.34-20171116

8 files changed, 243 insertions, 27 deletions

diff --git a/cmake/Package.cmake b/cmake/Package.cmake
index 83e2b53..c51eac0 100644
--- a/cmake/Package.cmake
+++ b/cmake/Package.cmake
@@ -37,6 +37,7 @@ endif()
 set(CPACK_RPM_USER_FILELIST "/usr/lib/systemd/system/mrenv.service"
                             "/usr/lib/systemd/system/mrzcpd.service"
                             "/usr/lib/systemd/system/mrtunnat.service"
+			    "/usr/lib/systemd/system/mrmonit.service"
                             "/etc/ld.so.conf.d/mrzcpd.conf"
                             "/etc/profile.d/mrzcpd.sh"
                             "${CMAKE_INSTALL_PREFIX}/lib/pkgconfig/mrzcpd.pc"
diff --git a/infra/include/common.h b/infra/include/common.h
index c4ed8c7..f01f6c9 100644
--- a/infra/include/common.h
+++ b/infra/include/common.h
@@ -286,7 +286,26 @@ errout:
     return RT_ERR;
 }
 
-/* =============================== G_VXLAN_DEFINE ===============================*/
+/* =============================== PROFILE ====================================== */
+static inline void timespec_diff(struct timespec * start, struct timespec * end,
+    struct timespec * result)
+{
+    if ((end->tv_nsec - start->tv_nsec) < 0)
+    {
+        result->tv_sec = end->tv_sec - start->tv_sec - 1;
+        result->tv_nsec = end->tv_nsec - start->tv_nsec + 1000000000;
+    }
+    else
+    {
+        result->tv_sec = end->tv_sec - start->tv_sec;
+        result->tv_nsec = end->tv_nsec - start->tv_nsec;
+    }
+    
+    return;
+}
+
+
+/* =============================== G_VXLAN_DEFINE =============================== */
 
 struct g_vxlan_hdr
 {
diff --git a/infra/src/ctrlmsg.c b/infra/src/ctrlmsg.c
index aedd4c1..2edca72 100644
--- a/infra/src/ctrlmsg.c
+++ b/infra/src/ctrlmsg.c
@@ -336,6 +336,15 @@ static void * __epoll_event_loop(void * arg)
                 strerror(errno)); goto errout;
         }
 
+#if 0
+        struct timespec __start_time;
+        struct timespec __end_time;
+        struct timespec __delay_time;
+
+        /* 计时，时间处理开始时间 */
+        clock_gettime(CLOCK_MONOTONIC, &__start_time);
+#endif
+
         for (int i = 0; i < ret; i++)
         {
             int fd = evlist[i].data.fd;
@@ -393,6 +402,17 @@ static void * __epoll_event_loop(void * arg)
         /* 超时事件，喂狗 */
         if (ret == 0 && timeout >= 0)
             __handle_timeout(handler);
+        
+#if 0
+
+        /* 计时，事件处理终止时间 */
+        clock_gettime(CLOCK_MONOTONIC, &__end_time);
+        timespec_diff(&__start_time, &__end_time, &__delay_time);
+
+        MR_DEBUG("Eventloop: event handle delay: %lld.%09ld",  
+            (long long)__delay_time.tv_sec,  __delay_time.tv_nsec);
+
+#endif
     }
 
 errout:
diff --git a/tools/monit_device/monit_device.py b/tools/monit_device/monit_device.py
index 364c9fa..e3c7300 100644
--- a/tools/monit_device/monit_device.py
+++ b/tools/monit_device/monit_device.py
@@ -7,6 +7,8 @@ import time
 import sys
 import signal
 import os
+import telegraf
+import socket
 
 TBPS = (1 * 1000 * 1000 * 1000 * 1000)
 GBPS = (1 * 1000 * 1000 * 1000)
@@ -92,6 +94,24 @@ def dump_human_table(json_fp, devsym, is_human_number = 0):
     table_phydev.add_row(SpeedList)
     print(table_phydev)
 
+# APM sendlog format
+def dump_apm_sendlog(json_fp, telegraf_client, devsym):
+
+    __metric_dict_speed = {}
+    __metric_dict_value = {}
+    __tags = {'device' : devsym }
+
+    for item in TITLE_VECTOR:
+        value = phydev_speed_read(json_fp, devsym, TITLE_MAP[item])
+        __metric_dict_speed[item] = value
+        
+    for item in TITLE_VECTOR:
+        value = phydev_value_read(json_fp, devsym, TITLE_MAP[item])
+        __metric_dict_value[item] = value
+
+    telegraf_client.metric('mr4_device_rxtx_speed', __metric_dict_speed, tags = __tags)
+    telegraf_client.metric('mr4_device_rxtx_value', __metric_dict_value, tags = __tags)
+    return
 
 def setup_argv_parser(phydev_list):
 
@@ -109,14 +129,60 @@ def setup_argv_parser(phydev_list):
     parser.add_argument('--clear-screen', help = 'clear screen at start of loop', 
                         action='store_true', default = 0)
 
+    # APM sendlog options
+    parser.add_argument('--sendlog-apm', help = 'send log to apm server',
+                        action='store_true', default = 0)
+    parser.add_argument('--sendlog-apm-cfg', help = 'send log configure file',
+                        type=str, default = '/opt/mrzcpd/etc/mrsendlog.conf')
+                        
     return parser.parse_args()
 
 def phydev_json_load():
     with open(G_JSON_PATH) as json_fp:
         return json.load(json_fp)
 
+
+def sendlog_hostname(test_hostname, test_port):
+    import socket
+    
+    hostname = socket.gethostname()
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    try:
+        s.connect((test_hostname, int(test_port)))
+        local_ip_addr = s.getsockname()[0]
+    except:
+        local_ip_addr = '127.0.0.1'
+    finally:
+        s.close()
+
+    return hostname, local_ip_addr
+
+def sendlog_apm_init(r_option):
+    import ConfigParser
+    import urllib
+
+    config = ConfigParser.ConfigParser()
+    config.read(r_option.sendlog_apm_cfg)
+
+    apm_server_url = config.get('sendlog_apm', 'apm_device_server')
+    url_proto, rest = urllib.splittype(apm_server_url)
+    url_host, rest = urllib.splithost(rest)
+    url_host, url_port = urllib.splitport(url_host)
+
+    hostname, local_ip_addr = sendlog_hostname(url_host, url_port)
+    sendlog_tags = {'host' : hostname, 'local_ip_addr': local_ip_addr}
+        
+    if url_proto == 'udp':
+        telegraf_client = telegraf.TelegrafClient(host = url_host, 
+            port = int(url_port), tags = sendlog_tags)
+    elif url_proto == 'http':
+        telegraf_client = telegraf.HttpClient(host = url_host, 
+            port = int(url_port), tags = sendlog_tags)
+
+    return telegraf_client
+
 def sigint_handler(handler, frame):
-    sys.exit(0)    
+    sys.exit(0)
 
 def main():
     signal.signal(signal.SIGINT, sigint_handler)
@@ -125,6 +191,10 @@ def main():
         json_fp = phydev_json_load()
         phydev_list = list_all_phydev(json_fp)
         r_option = setup_argv_parser(phydev_list)
+
+        if r_option.sendlog_apm:
+            telegraf_client = sendlog_apm_init(r_option)
+
         phydev_user_list = phydev_list if r_option.interface is None else r_option.interface
 
         while True:
@@ -132,8 +202,13 @@ def main():
                 os.system('clear')
 
             json_fp = phydev_json_load()
-            for devsym in phydev_user_list:
-                dump_human_table(json_fp, devsym, r_option.human_readable)
+
+            if r_option.sendlog_apm:
+               for devsym in phydev_user_list:
+                   dump_apm_sendlog(json_fp, telegraf_client, devsym)
+            else:
+               for devsym in phydev_user_list:
+                   dump_human_table(json_fp, devsym, r_option.human_readable)
 
             if not r_option.loop:
                 break
diff --git a/tools/monit_stream/monit_stream.py b/tools/monit_stream/monit_stream.py
index d0ff239..bbb8cfa 100644
--- a/tools/monit_stream/monit_stream.py
+++ b/tools/monit_stream/monit_stream.py
@@ -7,6 +7,7 @@ import time
 import sys
 import signal
 import os
+import telegraf
 
 TBPS = (1 * 1000 * 1000 * 1000 * 1000)
 GBPS = (1 * 1000 * 1000 * 1000)
@@ -31,7 +32,7 @@ TITLE_APP_MAP = {
     'PKTTx' : 'packet_send_count',
     'MbufAlloc' : 'mbuf_alloc_count',
     'MbufFree' : 'mbuf_free_count',
-    'MbufInUse' : 'mbuf_in_use_count'

+    'MbufInUse' : 'mbuf_in_use_count'
     }
 
 TITLE_MAP = { 'RxOnline'     : 'rx_on_line',
@@ -86,33 +87,33 @@ def vec_trans_to_human_readable(vec):
 
     return r_vector
 
-def dump_summary_table(json_fp, appsym, devsym, title_vector_rx, title_vector_tx,
-    is_human_number = 0, speed = 0):
+def dump_one_device(json_fp, devsym, title_vector_rx, title_vector_tx, speed):
 
-    def __dump_one_device(json_fp, devsym, title_vector_rx, title_vector_tx, speed):
+    __rd_function = vdev_value_read if speed == 0 else vdev_speed_read
 
-        __rd_function = vdev_value_read if speed == 0 else vdev_speed_read
+    ValueListSum = [0] * len(title_vector_rx + title_vector_tx)
+    nr_rxstream, nr_txstream = vdev_streams_read(json_fp, devsym)
 
-        ValueListSum = [0] * len(title_vector_rx + title_vector_tx)
-        nr_rxstream, nr_txstream = vdev_streams_read(json_fp, devsym)
+    for stream_id in range(max(nr_rxstream, nr_txstream)):
+        ValueList = []
 
-        for stream_id in range(max(nr_rxstream, nr_txstream)):
-            ValueList = []
+        for item in title_vector_rx:
+            value = __rd_function(json_fp, devsym, TITLE_MAP[item])[stream_id]  \
+                    if stream_id < nr_rxstream else 0
+            ValueList.append(value)
 
-            for item in title_vector_rx:
-                value = __rd_function(json_fp, devsym, TITLE_MAP[item])[stream_id]  \
-                        if stream_id < nr_rxstream else 0
-                ValueList.append(value)
+        for item in title_vector_tx:
+            value = __rd_function(json_fp, devsym, TITLE_MAP[item])[stream_id]  \
+                    if stream_id < nr_txstream else 0
+            ValueList.append(value)
 
-            for item in title_vector_tx:
-                value = __rd_function(json_fp, devsym, TITLE_MAP[item])[stream_id]  \
-                        if stream_id < nr_txstream else 0
-                ValueList.append(value)
+        for i,v in enumerate(ValueList):
+            ValueListSum[i] += v
 
-            for i,v in enumerate(ValueList):
-                ValueListSum[i] += v
+    return ValueListSum
 
-        return ValueListSum
+def dump_summary_table(json_fp, appsym, devsym, title_vector_rx, title_vector_tx,
+    is_human_number = 0, speed = 0):
 
     print('\nTime: %s, App: %s' % (time.strftime('%c'), appsym))
     table_phydev = prettytable.PrettyTable([' ']  + title_vector_rx + title_vector_tx,
@@ -124,7 +125,7 @@ def dump_summary_table(json_fp, appsym, devsym, title_vector_rx, title_vector_tx
     ValueListTotal = [0] * len(title_vector_rx + title_vector_tx)
 
     for dev in devsym:
-        ValueListSum = __dump_one_device(json_fp, dev, title_vector_rx, title_vector_tx, speed)
+        ValueListSum = dump_one_device(json_fp, dev, title_vector_rx, title_vector_tx, speed)
 
         for i,v in enumerate(ValueListSum):
             ValueListTotal[i] += v
@@ -189,7 +190,6 @@ def dump_human_table(json_fp, appsym, devsym,  title_vector_rx, title_vector_tx,
 
     print(table_phydev)
 
-
 def dump_status_table(json_fp, appsym):
     json_fp_appstat = json_fp['appstat']
     nr_stream = len(json_fp['appstat']['packet_recv_count'])
@@ -216,6 +216,25 @@ def dump_status_table(json_fp, appsym):
     table_phydev.add_row(['Total'] + ValueListSum)
     print(table_phydev)
 
+def dump_apm_sendlog(json_fp, telegraf_client, appsym, user_interface, title_vector_rx, title_vector_tx):
+    
+    for dev in user_interface:
+        ValueListSumSpeed = dump_one_device(json_fp, dev, title_vector_rx, title_vector_tx, 1)
+        ValueListSumValue = dump_one_device(json_fp, dev, title_vector_rx, title_vector_tx, 0)
+
+        sendlog_dict_speed = {}
+        sendlog_dict_value = {}
+        sendlog_tag = {'app': appsym, 'device': dev}
+
+        for id, value in enumerate(title_vector_rx + title_vector_tx):
+            sendlog_dict_speed[value] = ValueListSumSpeed[id]
+            sendlog_dict_value[value] = ValueListSumValue[id]
+
+        telegraf_client.metric('mr4_stream_rxtx_speed', sendlog_dict_speed, tags = sendlog_tag)
+        telegraf_client.metric('mr4_stream_rxtx_value', sendlog_dict_value, tags = sendlog_tag)
+
+    return
+
 
 def setup_argv_parser(applist):
 
@@ -243,6 +262,12 @@ def setup_argv_parser(applist):
     parser.add_argument('app', metavar='APP', help = 'the name of slave application', nargs = '*',
                         default=applist)
 
+    # APM sendlog options
+    parser.add_argument('--sendlog-apm', help = 'send log to apm server',
+                        action='store_true', default = 0)
+    parser.add_argument('--sendlog-apm-cfg', help = 'send log configure file',
+                        type=str, default = '/opt/mrzcpd/etc/mrsendlog.conf')
+
     return parser.parse_args()
 
 def global_json_load():
@@ -271,6 +296,45 @@ def check_vdev_options(json_fp, r_option):
             print("monit_stream: error: argument -i/--interface: invalid interface.")
             sys.exit(1)
 
+def sendlog_hostname(test_hostname, test_port):
+    import socket
+    
+    hostname = socket.gethostname()
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    try:
+        s.connect((test_hostname, int(test_port)))
+        local_ip_addr = s.getsockname()[0]
+    except:
+        local_ip_addr = '127.0.0.1'
+    finally:
+        s.close()
+
+    return hostname, local_ip_addr
+
+def sendlog_apm_init(r_option):
+    import ConfigParser
+    import urllib
+
+    config = ConfigParser.ConfigParser()
+    config.read(r_option.sendlog_apm_cfg)
+
+    apm_server_url = config.get('sendlog_apm', 'apm_stream_server')
+    url_proto, rest = urllib.splittype(apm_server_url)
+    url_host, rest = urllib.splithost(rest)
+    url_host, url_port = urllib.splitport(url_host)
+
+    hostname, local_ip_addr = sendlog_hostname(url_host, url_port)
+    sendlog_tags = {'host' : hostname, 'local_ip_addr': local_ip_addr}
+        
+    if url_proto == 'udp':
+        telegraf_client = telegraf.TelegrafClient(host = url_host, 
+            port = int(url_port), tags = sendlog_tags)
+    elif url_proto == 'http':
+        telegraf_client = telegraf.HttpClient(host = url_host, 
+            port = int(url_port), tags = sendlog_tags)
+
+    return telegraf_client
+
 def main():
     signal.signal(signal.SIGINT, sigint_handler)
 
@@ -301,6 +365,14 @@ def main():
         title_vector_tx.extend(TITLE_VECTOR_FTX)
 
     try:
+       if r_option.sendlog_apm:
+            telegraf_client = sendlog_apm_init(r_option)
+    except:
+       print("APM sendlog setup failed.")
+       raise
+       sys.exit(1)
+    
+    try:
         while True:
 
             if r_option.clear_screen:
@@ -315,6 +387,11 @@ def main():
                     dump_status_table(json_fp, appsym)
                     continue
 
+                if r_option.sendlog_apm:
+                    dump_apm_sendlog(json_fp, telegraf_client, appsym, user_interface, 
+                        title_vector_rx, title_vector_tx)
+                    continue
+
                 if not r_option.per_stream:
                    dump_summary_table(json_fp, appsym, user_interface, title_vector_rx, title_vector_tx,
                         r_option.human_readable, r_option.speed)
diff --git a/tools/systemd/CMakeLists.txt b/tools/systemd/CMakeLists.txt
index fe6a0d4..1f22299 100644
--- a/tools/systemd/CMakeLists.txt
+++ b/tools/systemd/CMakeLists.txt
@@ -1,6 +1,8 @@
 configure_file(mrenv.service.in mrenv.service)
 configure_file(mrzcpd.service.in mrzcpd.service)
 configure_file(mrtunnat.service.in mrtunnat.service)
+configure_file(mrmonit.service.in mrmonit.service)
+configure_file(mrmonit.timer.in mrmonit.timer)
 
 install(FILES ${CMAKE_BINARY_DIR}/tools/systemd/mrenv.service 
         DESTINATION ${MR_INSTALL_SYSUNITDIR} COMPONENT Program)
@@ -10,5 +12,11 @@ install(FILES ${CMAKE_BINARY_DIR}/tools/systemd/mrzcpd.service
 		
 install(FILES ${CMAKE_BINARY_DIR}/tools/systemd/mrtunnat.service 
         DESTINATION ${MR_INSTALL_SYSUNITDIR} COMPONENT Program)
+		
+install(FILES ${CMAKE_BINARY_DIR}/tools/systemd/mrmonit.service 
+        DESTINATION ${MR_INSTALL_SYSUNITDIR} COMPONENT Program)
+		
+install(FILES ${CMAKE_BINARY_DIR}/tools/systemd/mrmonit.timer 
+        DESTINATION ${MR_INSTALL_SYSUNITDIR} COMPONENT Program)
 
 install(PROGRAMS mrenv-script DESTINATION bin COMPONENT Program)
\ No newline at end of file
diff --git a/tools/systemd/mrmonit.service.in b/tools/systemd/mrmonit.service.in
new file mode 100644
index 0000000..80bf0e4
--- /dev/null
+++ b/tools/systemd/mrmonit.service.in
@@ -0,0 +1,16 @@
+[Unit]
+Description=Marsio APM logwritter
+After=network.target
+
+[Service]
+Type=simple
+ExecStartPre=/usr/bin/systemctl is-active mrzcpd
+ExecStartPre=-@MR_INSTALL_BINDIR@/monit_device --sendlog-apm
+ExecStartPre=-@MR_INSTALL_BINDIR@/monit_stream --sendlog-apm
+ExecStart=/bin/true
+StartLimitInterval=0
+RestartSec=1s
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file
diff --git a/tools/systemd/mrzcpd.service.in b/tools/systemd/mrzcpd.service.in
index 9b836fb..989553d 100644
--- a/tools/systemd/mrzcpd.service.in
+++ b/tools/systemd/mrzcpd.service.in
@@ -15,7 +15,7 @@ ExecStopPost=/usr/bin/rm -rf /run/.dpdk
 
 Restart=always
 RestartSec=5s
-WatchdogSec=5s
+WatchdogSec=10s
 Type=notify
 LimitCORE=infinity