From 42dded52ac976619359239e477f495f3fefd47ad Mon Sep 17 00:00:00 2001 From: luwenpeng Date: Mon, 19 Dec 2022 14:14:30 +0800 Subject: TSG-13075 TFE的Decrypted Traffic Steering功能增加第三方设备保活检查 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- conf/tfe/tfe.conf | 5 + platform/CMakeLists.txt | 2 +- platform/include/internal/proxy.h | 1 + platform/include/internal/watchdog_3rd_device.h | 8 ++ platform/src/acceptor_kni_v3.cpp | 3 +- platform/src/proxy.cpp | 5 + platform/src/tcp_stream.cpp | 14 ++- platform/src/watchdog_3rd_device.cpp | 144 ++++++++++++++++++++++++ 8 files changed, 174 insertions(+), 8 deletions(-) create mode 100644 platform/include/internal/watchdog_3rd_device.h create mode 100644 platform/src/watchdog_3rd_device.cpp diff --git a/conf/tfe/tfe.conf b/conf/tfe/tfe.conf index b5ba0a2..c09226e 100644 --- a/conf/tfe/tfe.conf +++ b/conf/tfe/tfe.conf @@ -176,6 +176,11 @@ so_mask_server=34 device_client=eth_client device_server=eth_server +http_keepalive_enable=1 +http_keepalive_path="/metrics" +http_keepalive_addr=192.168.41.60 +http_keepalive_port=9273 + [kafka] enable=1 vsystem_id=1 diff --git a/platform/CMakeLists.txt b/platform/CMakeLists.txt index c6c34f2..a065039 100644 --- a/platform/CMakeLists.txt +++ b/platform/CMakeLists.txt @@ -4,7 +4,7 @@ find_package(NFNETLINK REQUIRED) add_executable(tfe src/acceptor_kni_v1.cpp src/acceptor_kni_v2.cpp src/acceptor_kni_v3.cpp src/ssl_stream.cpp src/key_keeper.cpp src/ssl_fetch_cert.cpp src/ssl_sess_cache.cpp src/ssl_sess_ticket.cpp src/ssl_service_cache.cpp src/ssl_trusted_cert_storage.cpp src/ev_root_ca_metadata.cpp src/ssl_utils.cpp - src/tcp_stream.cpp src/main.cpp src/proxy.cpp src/sender_scm.cpp src/watchdog_kni.cpp src/watchdog_tfe.cpp src/ssl_ja3.cpp) + src/tcp_stream.cpp src/main.cpp src/proxy.cpp src/sender_scm.cpp src/watchdog_kni.cpp src/watchdog_tfe.cpp src/ssl_ja3.cpp src/watchdog_3rd_device.cpp) target_include_directories(tfe PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/external) target_include_directories(tfe PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include/internal) diff --git a/platform/include/internal/proxy.h b/platform/include/internal/proxy.h index 970a6d1..aeecee5 100644 --- a/platform/include/internal/proxy.h +++ b/platform/include/internal/proxy.h @@ -129,6 +129,7 @@ struct tfe_proxy struct sender_scm * scm_sender; struct watchdog_kni * watchdog_kni; struct watchdog_tfe * watchdog_tfe; + struct watchdog_3rd_device * watchdog_3rd_device; /* DEBUG OPTIONS */ unsigned int tcp_all_passthrough; diff --git a/platform/include/internal/watchdog_3rd_device.h b/platform/include/internal/watchdog_3rd_device.h new file mode 100644 index 0000000..12727e7 --- /dev/null +++ b/platform/include/internal/watchdog_3rd_device.h @@ -0,0 +1,8 @@ +#ifndef TFE_WATCHDOG_3RD_H +#define TFE_WATCHDOG_3RD_H + +struct watchdog_3rd_device; +struct watchdog_3rd_device *watchdog_3rd_device_create(struct tfe_proxy *proxy, const char *profile, void *logger); +int steering_device_is_available(); + +#endif \ No newline at end of file diff --git a/platform/src/acceptor_kni_v3.cpp b/platform/src/acceptor_kni_v3.cpp index 1045c13..ac9f4f9 100644 --- a/platform/src/acceptor_kni_v3.cpp +++ b/platform/src/acceptor_kni_v3.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #define TCP_RESTORE_TCPOPT_KIND 88 @@ -421,7 +422,7 @@ static int payload_handler_cb(struct nfq_q_handle *qh, struct nfgenmsg *nfmsg, s goto end; } - if (__ctx->proxy->traffic_steering_options.enable) + if (__ctx->proxy->traffic_steering_options.enable && steering_device_is_available()) { fd_fake_c = tfe_tcp_restore_fd_create(&(restore_info.client), &(restore_info.server), __ctx->proxy->traffic_steering_options.device_client, __ctx->proxy->traffic_steering_options.so_mask_client); if (fd_fake_c < 0) diff --git a/platform/src/proxy.cpp b/platform/src/proxy.cpp index a162e23..31fcf97 100644 --- a/platform/src/proxy.cpp +++ b/platform/src/proxy.cpp @@ -50,6 +50,7 @@ #include #include #include +#include /* Breakpad */ #include @@ -696,6 +697,10 @@ int main(int argc, char * argv[]) g_default_proxy->watchdog_tfe = watchdog_tfe_create(g_default_proxy, main_profile, g_default_logger); CHECK_OR_EXIT(g_default_proxy->watchdog_tfe != NULL, "Failed at creating TFE watchdog, Exit."); + /* Watchdog 3rd device */ + g_default_proxy->watchdog_3rd_device = watchdog_3rd_device_create(g_default_proxy, main_profile, g_default_logger); + CHECK_OR_EXIT(g_default_proxy->watchdog_3rd_device != NULL, "Failed at creating 3rd device watchdog, Exit."); + TFE_LOG_ERROR(g_default_logger, "Tango Frontend Engine initialized, Version: %s.", __tfe_version); /* If TFE is run by systemd's notify, then tell the systemd our tfe is ready. diff --git a/platform/src/tcp_stream.cpp b/platform/src/tcp_stream.cpp index 0c13deb..7b63a13 100644 --- a/platform/src/tcp_stream.cpp +++ b/platform/src/tcp_stream.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #ifndef TFE_CONFIG_OUTPUT_LIMIT_DEFAULT #define TFE_CONFIG_OUTPUT_LIMIT_DEFAULT (1024 * 1024) @@ -588,7 +589,7 @@ static void __stream_bev_readcb(struct bufferevent * bev, void * arg) struct evbuffer * inbuf = NULL; struct evbuffer * outbuf = NULL; - if (_stream->proxy_ref->traffic_steering_options.enable) + if (_stream->proxy_ref->traffic_steering_options.enable && steering_device_is_available()) { if (bev == _stream->conn_downstream->bev) { @@ -773,7 +774,7 @@ static void __stream_bev_writecb(struct bufferevent * bev, void * arg) struct tfe_conn_private ** ref_peer_conn{}; struct ssl_stream ** ref_this_ssl_stream{}; - if (_stream->proxy_ref->traffic_steering_options.enable) + if (_stream->proxy_ref->traffic_steering_options.enable && steering_device_is_available()) { // TODO 增加计数 TFE_LOG_DEBUG(__STREAM_LOGGER(_stream), "decrypted traffic steering, %s run writecb", bev == _stream->conn_downstream->bev ? "conn_downstream" : "conn_upstream"); @@ -836,7 +837,7 @@ static void __stream_bev_eventcb(struct bufferevent * bev, short events, void * enum tfe_conn_dir peer_conn_dir{}; size_t rx_offset = 0; - if (_stream->proxy_ref->traffic_steering_options.enable) + if (_stream->proxy_ref->traffic_steering_options.enable && steering_device_is_available()) { // TODO 增加计数 TFE_LOG_DEBUG(__STREAM_LOGGER(_stream), "decrypted traffic steering, %s run eventcb", bev == _stream->conn_downstream->bev ? "conn_downstream" : "conn_upstream"); @@ -1251,7 +1252,7 @@ void ssl_downstream_create_on_success(future_result_t * result, void * user) __conn_private_enable(_stream->conn_downstream); __conn_private_enable(_stream->conn_upstream); - if (_stream->proxy_ref->traffic_steering_options.enable) + if (_stream->proxy_ref->traffic_steering_options.enable && steering_device_is_available()) { __conn_private_enable(_stream->conn_fake_c); __conn_private_enable(_stream->conn_fake_s); @@ -1418,6 +1419,7 @@ void tfe_stream_destory(struct tfe_stream_private * stream) if (__is_ssl(stream) && stream->ssl_downstream) { + // TODO core dump ssl_stream_free(stream->ssl_downstream, ev_base, stream->conn_downstream->bev); } @@ -1754,7 +1756,7 @@ int tfe_stream_init_by_fds(struct tfe_stream * stream, evutil_socket_t fd_downst __stream_fd_option_setup(_stream, fd_downstream, CONN_DIR_DOWNSTREAM); __stream_fd_option_setup(_stream, fd_upstream, CONN_DIR_UPSTREAM); - if (_stream->proxy_ref->traffic_steering_options.enable) + if (_stream->proxy_ref->traffic_steering_options.enable && steering_device_is_available()) { __stream_fd_option_setup(_stream, fd_fake_s, CONN_DIR_DOWNSTREAM); __stream_fd_option_setup(_stream, fd_fake_c, CONN_DIR_UPSTREAM); @@ -1811,7 +1813,7 @@ int tfe_stream_init_by_fds(struct tfe_stream * stream, evutil_socket_t fd_downst __conn_private_enable(_stream->conn_downstream); __conn_private_enable(_stream->conn_upstream); - if (_stream->proxy_ref->traffic_steering_options.enable) + if (_stream->proxy_ref->traffic_steering_options.enable && steering_device_is_available()) { __conn_private_enable(_stream->conn_fake_s); __conn_private_enable(_stream->conn_fake_c); diff --git a/platform/src/watchdog_3rd_device.cpp b/platform/src/watchdog_3rd_device.cpp new file mode 100644 index 0000000..56135bd --- /dev/null +++ b/platform/src/watchdog_3rd_device.cpp @@ -0,0 +1,144 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct watchdog_3rd_device +{ + struct tfe_proxy *proxy; + void *logger; + const char *profile; + pthread_t pthread; + + unsigned int http_keepalive_enable; + unsigned int http_keepalive_port; + char http_keepalive_path[TFE_PATH_MAX]; + char http_keepalive_addr[TFE_SYMBOL_MAX]; +}; + +static long long g_steering_device_is_available = 1; + +static int http_keepalive(struct watchdog_3rd_device *__ctx) +{ + static char req_buff[1024] = { 0 }; + static char rsp_buff[4096] = { 0 }; + struct sockaddr_in addr; + const char *expect= "HTTP/1.1 200 OK"; + char http_head[] = + "GET %s HTTP/1.1\r\n" + "Host: %s:%d\r\n" + "User-Agent: curl/7.47.0\r\n" + "Accept: */*\r\n\r\n"; + sprintf(req_buff, http_head, __ctx->http_keepalive_path, __ctx->http_keepalive_addr, __ctx->http_keepalive_port); + + int sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd == -1) + { + TFE_LOG_ERROR(__ctx->logger, "Watchdog 3rd thread fail to create socket(), %s", strerror(errno)); + goto error; + } + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(__ctx->http_keepalive_port); + addr.sin_addr.s_addr = inet_addr(__ctx->http_keepalive_addr); + if (connect(sockfd, (const struct sockaddr *)&addr, sizeof(addr)) == -1) + { + TFE_LOG_ERROR(__ctx->logger, "Watchdog 3rd thread fail to connect(), %s", strerror(errno)); + goto error; + } + + if (write(sockfd, req_buff, strlen(req_buff)) == -1) + { + TFE_LOG_ERROR(__ctx->logger, "Watchdog 3rd thread fail to write(), %s", strerror(errno)); + goto error; + } + + if (read(sockfd, rsp_buff, sizeof(rsp_buff)) == -1) + { + TFE_LOG_ERROR(__ctx->logger, "Watchdog 3rd thread fail to read(), %s", strerror(errno)); + goto error; + } + + if (strncmp(rsp_buff, expect, strlen(expect)) != 0) + { + TFE_LOG_ERROR(__ctx->logger, "Watchdog 3rd thread get unexpect http response"); + goto error; + } + close(sockfd); + + return 0; + +error: + if (sockfd) + { + close(sockfd); + } + + return -1; +} + +static void * watchdog_3rd_device_thread(void * arg) +{ + struct watchdog_3rd_device * __ctx = (struct watchdog_3rd_device *)arg; + + char thread_name[16]; + snprintf(thread_name, sizeof(thread_name), "watchdog:3rd"); + prctl(PR_SET_NAME, (unsigned long long)thread_name, NULL, NULL, NULL); + + while (1) + { + if (http_keepalive(__ctx) == -1) + { + ATOMIC_ZERO(&g_steering_device_is_available); + TFE_LOG_ERROR(__ctx->logger, "Watchdog 3rd thread on fail: 3rd steering device is unavailable !!!"); + } + else + { + ATOMIC_INC(&g_steering_device_is_available); + } + sleep(1); + } + + TFE_LOG_ERROR(__ctx->logger, "Watchdog 3rd thread exit"); + + return NULL; +} + +int steering_device_is_available() +{ + return ATOMIC_READ(&g_steering_device_is_available); +} + +struct watchdog_3rd_device *watchdog_3rd_device_create(struct tfe_proxy *proxy, const char *profile, void *logger) +{ + struct watchdog_3rd_device *__ctx = ALLOC(struct watchdog_3rd_device, 1); + + __ctx->proxy = proxy; + __ctx->logger = logger; + __ctx->profile = profile; + + MESA_load_profile_uint_def(profile, "traffic_steering", "http_keepalive_enable", &(__ctx->http_keepalive_enable), 0); + MESA_load_profile_uint_def(profile, "traffic_steering", "http_keepalive_port", &(__ctx->http_keepalive_port), 80); + MESA_load_profile_string_def(profile, "traffic_steering", "http_keepalive_path", __ctx->http_keepalive_path, sizeof(__ctx->http_keepalive_path), "/"); + MESA_load_profile_string_def(profile, "traffic_steering", "http_keepalive_addr", __ctx->http_keepalive_addr, sizeof(__ctx->http_keepalive_addr), "127.0.0.1"); + if (!__ctx->http_keepalive_enable) + { + return __ctx; + } + + if (pthread_create(&__ctx->pthread, NULL, watchdog_3rd_device_thread, (void *)__ctx) < 0) + { + TFE_LOG_ERROR(__ctx->logger, "Fail to create 3rd watchdog thread: %s", strerror(errno)); + return NULL; + } + + TFE_LOG_INFO(__ctx->logger, "Watchdog 3rd module init successfully."); + return __ctx; +}; \ No newline at end of file -- cgit v1.2.3