TSG-13075 TFE的Decrypted Traffic Steering功能增加第三方设备保活检查

8 files changed, 174 insertions, 8 deletions

diff --git a/conf/tfe/tfe.conf b/conf/tfe/tfe.conf
index b5ba0a2..c09226e 100644
--- a/conf/tfe/tfe.conf
+++ b/conf/tfe/tfe.conf
@@ -176,6 +176,11 @@ so_mask_server=34
 device_client=eth_client
 device_server=eth_server
 
+http_keepalive_enable=1
+http_keepalive_path="/metrics"
+http_keepalive_addr=192.168.41.60
+http_keepalive_port=9273
+
 [kafka]
 enable=1
 vsystem_id=1
diff --git a/platform/CMakeLists.txt b/platform/CMakeLists.txt
index c6c34f2..a065039 100644
--- a/platform/CMakeLists.txt
+++ b/platform/CMakeLists.txt
@@ -4,7 +4,7 @@ find_package(NFNETLINK REQUIRED)
 add_executable(tfe src/acceptor_kni_v1.cpp src/acceptor_kni_v2.cpp src/acceptor_kni_v3.cpp src/ssl_stream.cpp src/key_keeper.cpp src/ssl_fetch_cert.cpp
 	src/ssl_sess_cache.cpp src/ssl_sess_ticket.cpp src/ssl_service_cache.cpp
 	src/ssl_trusted_cert_storage.cpp src/ev_root_ca_metadata.cpp src/ssl_utils.cpp
-	src/tcp_stream.cpp src/main.cpp src/proxy.cpp src/sender_scm.cpp src/watchdog_kni.cpp src/watchdog_tfe.cpp src/ssl_ja3.cpp)
+	src/tcp_stream.cpp src/main.cpp src/proxy.cpp src/sender_scm.cpp src/watchdog_kni.cpp src/watchdog_tfe.cpp src/ssl_ja3.cpp src/watchdog_3rd_device.cpp)
 
 target_include_directories(tfe PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/external)
 target_include_directories(tfe PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include/internal)
diff --git a/platform/include/internal/proxy.h b/platform/include/internal/proxy.h
index 970a6d1..aeecee5 100644
--- a/platform/include/internal/proxy.h
+++ b/platform/include/internal/proxy.h
@@ -129,6 +129,7 @@ struct tfe_proxy
     struct sender_scm * scm_sender;
     struct watchdog_kni * watchdog_kni;
 	struct watchdog_tfe * watchdog_tfe;
+	struct watchdog_3rd_device * watchdog_3rd_device;
 
 	/* DEBUG OPTIONS */
 	unsigned int tcp_all_passthrough;
diff --git a/platform/include/internal/watchdog_3rd_device.h b/platform/include/internal/watchdog_3rd_device.h
new file mode 100644
index 0000000..12727e7
--- /dev/null
+++ b/platform/include/internal/watchdog_3rd_device.h
@@ -0,0 +1,8 @@
+#ifndef TFE_WATCHDOG_3RD_H
+#define TFE_WATCHDOG_3RD_H
+
+struct watchdog_3rd_device;
+struct watchdog_3rd_device *watchdog_3rd_device_create(struct tfe_proxy *proxy, const char *profile, void *logger);
+int steering_device_is_available();
+
+#endif
\ No newline at end of file
diff --git a/platform/src/acceptor_kni_v3.cpp b/platform/src/acceptor_kni_v3.cpp
index 1045c13..ac9f4f9 100644
--- a/platform/src/acceptor_kni_v3.cpp
+++ b/platform/src/acceptor_kni_v3.cpp
@@ -12,6 +12,7 @@
 #include <tfe_pkt_util.h>
 #include <tfe_tcp_restore.h>
 #include <MESA/MESA_prof_load.h>
+#include <watchdog_3rd_device.h>
 
 #define TCP_RESTORE_TCPOPT_KIND 88
 
@@ -421,7 +422,7 @@ static int payload_handler_cb(struct nfq_q_handle *qh, struct nfgenmsg *nfmsg, s
 		goto end;
 	}
 
-	if (__ctx->proxy->traffic_steering_options.enable)
+	if (__ctx->proxy->traffic_steering_options.enable && steering_device_is_available())
 	{
 		fd_fake_c = tfe_tcp_restore_fd_create(&(restore_info.client), &(restore_info.server), __ctx->proxy->traffic_steering_options.device_client, __ctx->proxy->traffic_steering_options.so_mask_client);
 		if (fd_fake_c < 0)
diff --git a/platform/src/proxy.cpp b/platform/src/proxy.cpp
index a162e23..31fcf97 100644
--- a/platform/src/proxy.cpp
+++ b/platform/src/proxy.cpp
@@ -50,6 +50,7 @@
 #include <watchdog_kni.h>
 #include <watchdog_tfe.h>
 #include <key_keeper.h>
+#include <watchdog_3rd_device.h>
 
 /* Breakpad */
 #include <MESA/breakpad_mini.h>
@@ -696,6 +697,10 @@ int main(int argc, char * argv[])
 	g_default_proxy->watchdog_tfe = watchdog_tfe_create(g_default_proxy, main_profile, g_default_logger);
 	CHECK_OR_EXIT(g_default_proxy->watchdog_tfe != NULL, "Failed at creating TFE watchdog, Exit.");
 
+	/* Watchdog 3rd device */
+	g_default_proxy->watchdog_3rd_device = watchdog_3rd_device_create(g_default_proxy, main_profile, g_default_logger);
+	CHECK_OR_EXIT(g_default_proxy->watchdog_3rd_device != NULL, "Failed at creating 3rd device watchdog, Exit.");
+
 	TFE_LOG_ERROR(g_default_logger, "Tango Frontend Engine initialized, Version: %s.", __tfe_version);
 
 	/* If TFE is run by systemd's notify, then tell the systemd our tfe is ready.
diff --git a/platform/src/tcp_stream.cpp b/platform/src/tcp_stream.cpp
index 0c13deb..7b63a13 100644
--- a/platform/src/tcp_stream.cpp
+++ b/platform/src/tcp_stream.cpp
@@ -33,6 +33,7 @@
 #include <proxy.h>
 #include <netinet/tcp.h>
 #include <event2/bufferevent_struct.h>
+#include <watchdog_3rd_device.h>
 
 #ifndef TFE_CONFIG_OUTPUT_LIMIT_DEFAULT
 #define TFE_CONFIG_OUTPUT_LIMIT_DEFAULT                (1024 * 1024)
@@ -588,7 +589,7 @@ static void __stream_bev_readcb(struct bufferevent * bev, void * arg)
 	struct evbuffer * inbuf = NULL;
 	struct evbuffer * outbuf = NULL;
 
-	if (_stream->proxy_ref->traffic_steering_options.enable)
+	if (_stream->proxy_ref->traffic_steering_options.enable && steering_device_is_available())
 	{
 		if (bev == _stream->conn_downstream->bev)
 		{
@@ -773,7 +774,7 @@ static void __stream_bev_writecb(struct bufferevent * bev, void * arg)
 	struct tfe_conn_private ** ref_peer_conn{};
 	struct ssl_stream ** ref_this_ssl_stream{};
 
-	if (_stream->proxy_ref->traffic_steering_options.enable)
+	if (_stream->proxy_ref->traffic_steering_options.enable && steering_device_is_available())
 	{
 		// TODO 增加计数
 		TFE_LOG_DEBUG(__STREAM_LOGGER(_stream), "decrypted traffic steering, %s run writecb",  bev == _stream->conn_downstream->bev ? "conn_downstream" : "conn_upstream");
@@ -836,7 +837,7 @@ static void __stream_bev_eventcb(struct bufferevent * bev, short events, void *
 	enum tfe_conn_dir peer_conn_dir{};
 	size_t rx_offset = 0;
 
-	if (_stream->proxy_ref->traffic_steering_options.enable)
+	if (_stream->proxy_ref->traffic_steering_options.enable && steering_device_is_available())
 	{
 		// TODO 增加计数
 		TFE_LOG_DEBUG(__STREAM_LOGGER(_stream), "decrypted traffic steering, %s run eventcb",  bev == _stream->conn_downstream->bev ? "conn_downstream" : "conn_upstream");
@@ -1251,7 +1252,7 @@ void ssl_downstream_create_on_success(future_result_t * result, void * user)
 	__conn_private_enable(_stream->conn_downstream);
 	__conn_private_enable(_stream->conn_upstream);
 
-	if (_stream->proxy_ref->traffic_steering_options.enable)
+	if (_stream->proxy_ref->traffic_steering_options.enable && steering_device_is_available())
 	{
 		__conn_private_enable(_stream->conn_fake_c);
 		__conn_private_enable(_stream->conn_fake_s);
@@ -1418,6 +1419,7 @@ void tfe_stream_destory(struct tfe_stream_private * stream)
 
 	if (__is_ssl(stream) && stream->ssl_downstream)
 	{
+		// TODO core dump
 		ssl_stream_free(stream->ssl_downstream, ev_base, stream->conn_downstream->bev);
 	}
 
@@ -1754,7 +1756,7 @@ int tfe_stream_init_by_fds(struct tfe_stream * stream, evutil_socket_t fd_downst
 	__stream_fd_option_setup(_stream, fd_downstream, CONN_DIR_DOWNSTREAM);
 	__stream_fd_option_setup(_stream, fd_upstream, CONN_DIR_UPSTREAM);
 
-	if (_stream->proxy_ref->traffic_steering_options.enable)
+	if (_stream->proxy_ref->traffic_steering_options.enable && steering_device_is_available())
 	{
 		__stream_fd_option_setup(_stream, fd_fake_s, CONN_DIR_DOWNSTREAM);
 		__stream_fd_option_setup(_stream, fd_fake_c, CONN_DIR_UPSTREAM);
@@ -1811,7 +1813,7 @@ int tfe_stream_init_by_fds(struct tfe_stream * stream, evutil_socket_t fd_downst
 		__conn_private_enable(_stream->conn_downstream);
 		__conn_private_enable(_stream->conn_upstream);
 
-		if (_stream->proxy_ref->traffic_steering_options.enable)
+		if (_stream->proxy_ref->traffic_steering_options.enable && steering_device_is_available())
 		{
 			__conn_private_enable(_stream->conn_fake_s);
 			__conn_private_enable(_stream->conn_fake_c);
diff --git a/platform/src/watchdog_3rd_device.cpp b/platform/src/watchdog_3rd_device.cpp
new file mode 100644
index 0000000..56135bd
--- /dev/null
+++ b/platform/src/watchdog_3rd_device.cpp
@@ -0,0 +1,144 @@
+#include <arpa/inet.h>
+#include <netinet/tcp.h>
+#include <unistd.h>
+#include <assert.h>
+#include <sys/prctl.h>
+#include <stdlib.h>
+#include <proxy.h>
+#include <platform.h>
+#include <tfe_utils.h>
+#include <watchdog_3rd_device.h>
+#include <MESA/MESA_prof_load.h>
+
+struct watchdog_3rd_device
+{
+    struct tfe_proxy *proxy;
+    void *logger;
+    const char *profile;
+    pthread_t pthread;
+
+    unsigned int http_keepalive_enable;
+    unsigned int http_keepalive_port;
+	char http_keepalive_path[TFE_PATH_MAX];
+    char http_keepalive_addr[TFE_SYMBOL_MAX];
+};
+
+static long long g_steering_device_is_available = 1;
+
+static int http_keepalive(struct watchdog_3rd_device *__ctx)
+{
+	static char req_buff[1024] = { 0 };
+	static char rsp_buff[4096] = { 0 };
+	struct sockaddr_in addr;
+    const char *expect= "HTTP/1.1 200 OK";
+	char http_head[] =
+    	"GET %s HTTP/1.1\r\n"
+    	"Host: %s:%d\r\n"
+    	"User-Agent: curl/7.47.0\r\n"
+    	"Accept: */*\r\n\r\n";
+	sprintf(req_buff, http_head, __ctx->http_keepalive_path, __ctx->http_keepalive_addr, __ctx->http_keepalive_port);
+
+	int sockfd = socket(AF_INET, SOCK_STREAM, 0);
+    if (sockfd == -1)
+	{
+		TFE_LOG_ERROR(__ctx->logger, "Watchdog 3rd thread fail to create socket(), %s", strerror(errno));
+		goto error; 
+	}
+    memset(&addr, 0, sizeof(addr));
+    addr.sin_family = AF_INET;
+    addr.sin_port = htons(__ctx->http_keepalive_port);
+    addr.sin_addr.s_addr = inet_addr(__ctx->http_keepalive_addr);
+    if (connect(sockfd, (const struct sockaddr *)&addr, sizeof(addr)) == -1)
+    {
+        TFE_LOG_ERROR(__ctx->logger, "Watchdog 3rd thread fail to connect(), %s", strerror(errno));
+		goto error; 
+    }
+
+	if (write(sockfd, req_buff, strlen(req_buff)) == -1)
+	{
+        TFE_LOG_ERROR(__ctx->logger, "Watchdog 3rd thread fail to write(), %s", strerror(errno));
+		goto error; 
+	}
+
+	if (read(sockfd, rsp_buff, sizeof(rsp_buff)) == -1)
+	{
+        TFE_LOG_ERROR(__ctx->logger, "Watchdog 3rd thread fail to read(), %s", strerror(errno));
+		goto error; 
+	}
+
+    if (strncmp(rsp_buff, expect, strlen(expect)) != 0)
+    {
+        TFE_LOG_ERROR(__ctx->logger, "Watchdog 3rd thread get unexpect http response");
+		goto error; 
+    }
+	close(sockfd);
+	
+    return 0;
+
+error:
+	if (sockfd)
+    {
+		close(sockfd);
+    }
+
+	return -1;
+}
+
+static void * watchdog_3rd_device_thread(void * arg)
+{
+    struct watchdog_3rd_device * __ctx = (struct watchdog_3rd_device *)arg;
+
+	char thread_name[16];
+    snprintf(thread_name, sizeof(thread_name), "watchdog:3rd");
+    prctl(PR_SET_NAME, (unsigned long long)thread_name, NULL, NULL, NULL);
+
+	while (1)
+	{
+		if (http_keepalive(__ctx) == -1)
+		{
+            ATOMIC_ZERO(&g_steering_device_is_available);
+			TFE_LOG_ERROR(__ctx->logger, "Watchdog 3rd thread on fail: 3rd steering device is unavailable !!!");
+		}
+		else
+		{
+            ATOMIC_INC(&g_steering_device_is_available);
+		}
+		sleep(1);
+	}
+
+	TFE_LOG_ERROR(__ctx->logger, "Watchdog 3rd thread exit");
+
+	return NULL;
+}
+
+int steering_device_is_available()
+{
+    return ATOMIC_READ(&g_steering_device_is_available);
+}
+
+struct watchdog_3rd_device *watchdog_3rd_device_create(struct tfe_proxy *proxy, const char *profile, void *logger)
+{
+    struct watchdog_3rd_device *__ctx = ALLOC(struct watchdog_3rd_device, 1);
+
+    __ctx->proxy = proxy;
+    __ctx->logger = logger;
+    __ctx->profile = profile;
+
+    MESA_load_profile_uint_def(profile, "traffic_steering", "http_keepalive_enable", &(__ctx->http_keepalive_enable), 0);
+    MESA_load_profile_uint_def(profile, "traffic_steering", "http_keepalive_port", &(__ctx->http_keepalive_port), 80);
+    MESA_load_profile_string_def(profile, "traffic_steering", "http_keepalive_path", __ctx->http_keepalive_path, sizeof(__ctx->http_keepalive_path), "/");
+    MESA_load_profile_string_def(profile, "traffic_steering", "http_keepalive_addr", __ctx->http_keepalive_addr, sizeof(__ctx->http_keepalive_addr), "127.0.0.1");
+    if (!__ctx->http_keepalive_enable)
+    {
+        return __ctx;
+    }
+
+    if (pthread_create(&__ctx->pthread, NULL, watchdog_3rd_device_thread, (void *)__ctx) < 0)
+    {
+        TFE_LOG_ERROR(__ctx->logger, "Fail to create 3rd watchdog thread: %s", strerror(errno));
+        return NULL;
+    }
+
+    TFE_LOG_INFO(__ctx->logger, "Watchdog 3rd module init successfully.");
+    return __ctx;
+};
\ No newline at end of file