perf: 性能优化

    * io_uring使用buffer pool避免内存分配与释放
    * packet io thread与worker thread无锁访问cmsg
    * 为解密流量的fd设置默认的TTL

4 files changed, 72 insertions, 338 deletions

diff --git a/platform/src/acceptor_kni_v3.cpp b/platform/src/acceptor_kni_v3.cpp
index ac50f3b..2011c78 100644
--- a/platform/src/acceptor_kni_v3.cpp
+++ b/platform/src/acceptor_kni_v3.cpp
@@ -13,7 +13,6 @@
 #include <tfe_tcp_restore.h>
 #include <MESA/MESA_prof_load.h>
 #include <watchdog_3rd_device.h>
-#include <raw_socket.h>
 #include <packet_construct.h>
 #include <intercept_policy.h>
 
@@ -282,257 +281,6 @@ static void tcp_restore_info_parse_from_pkt(struct pkt_info *pktinfo, struct tcp
 	}
 }
 
-struct tcp_option_mss {
-	uint8_t kind;
-	uint8_t length;
-	uint16_t mss_value;
-} __attribute__((__packed__));
-
-struct tcp_option_window_scale {
-	uint8_t kind;
-	uint8_t length;
-	uint8_t shift_count;
-} __attribute__((__packed__));
-
-struct tcp_option_sack {
-	uint8_t kind;
-	uint8_t length;
-} __attribute__((__packed__));
-
-struct tcp_option_time_stamp {
-	uint8_t kind;
-	uint8_t length;
-	uint32_t tsval;
-	uint32_t tsecr;
-} __attribute__((__packed__));
-
-static int fake_tcp_handshake(struct tfe_proxy *proxy, struct tcp_restore_info *restore_info)
-{
-    char buffer[1500] = {0};
-    int length = 0;
-
-	char tcp_option_buffer_c[40] = {0};
-	char tcp_option_buffer_s[40] = {0};
-	char tcp_option_buffer_c2[40] = {0};
-	int tcp_option_length_c = 0;
-	int tcp_option_length_s = 0;
-	int tcp_option_length_c2 = 0;
-
-	const struct tcp_restore_endpoint *client = &restore_info->client;
-	const struct tcp_restore_endpoint *server = &restore_info->server;
-    struct raw_socket *raw_socket_c = raw_socket_create(proxy->traffic_steering_options.device_client, proxy->traffic_steering_options.so_mask_client);
-    struct raw_socket *raw_socket_s = raw_socket_create(proxy->traffic_steering_options.device_server, proxy->traffic_steering_options.so_mask_server);
-    if (raw_socket_c == NULL || raw_socket_s == NULL)
-    {
-    	raw_socket_destory(raw_socket_c);
-    	raw_socket_destory(raw_socket_s);
-
-    	return -1;
-    }
-
-	uint32_t c_seq = client->seq - 1;
-	uint32_t s_seq = server->seq - 1;
-
-	/*
-	 * Maximum segment size: Kind: 2, Length: 4
-	 * +---------+---------+---------+
-	 * | Kind=2  |Length=4 |mss.value|
-	 * +---------+---------+---------+
-	 *      1         1         2
-	 */
-	if (client->mss && server->mss)
-	{
-		struct tcp_option_mss *option_c = (struct tcp_option_mss *)(tcp_option_buffer_c + tcp_option_length_c);
-		option_c->kind = 2;
-		option_c->length = 4;
-		option_c->mss_value = htons(client->mss);
-		tcp_option_length_c += sizeof(struct tcp_option_mss);
-
-		struct tcp_option_mss *option_s = (struct tcp_option_mss *)(tcp_option_buffer_s + tcp_option_length_s);
-		option_s->kind = 2;
-		option_s->length = 4;
-		option_s->mss_value = htons(server->mss);
-		tcp_option_length_s += sizeof(struct tcp_option_mss);
-	}
-
-	/*
-	 * Window Scale option: Kind: 3, Length: 3
-	 * +---------+---------+---------+
-	 * | Kind=3  |Length=3 |shift.cnt|
-	 * +---------+---------+---------+
-	 *      1         1         1
-	 */
-	if (client->wscale_perm && server->wscale_perm)
-	{
-		// padding
-		memset(tcp_option_buffer_c + tcp_option_length_c, 1, 1);
-		tcp_option_length_c += 1;
-		memset(tcp_option_buffer_s + tcp_option_length_s, 1, 1);
-		tcp_option_length_s += 1;
-
-		struct tcp_option_window_scale *option_c = (struct tcp_option_window_scale *)(tcp_option_buffer_c + tcp_option_length_c);
-		option_c->kind = 3;
-		option_c->length = 3;
-		option_c->shift_count = client->wscale;
-		tcp_option_length_c += sizeof(struct tcp_option_window_scale);
-
-		struct tcp_option_window_scale *option_s = (struct tcp_option_window_scale *)(tcp_option_buffer_s + tcp_option_length_s);
-		option_s->kind = 3;
-		option_s->length = 3;
-		option_s->shift_count = server->wscale;
-		tcp_option_length_s += sizeof(struct tcp_option_window_scale);
-	}
-
-	/*
-	 * SACK option: Kind: 4, Length: 2
-	 * +---------+---------+
-	 * | Kind=4  |Length=2 |
-	 * +---------+---------+
-	 *      1         1
-	 */
-	if (client->sack_perm && server->sack_perm)
-	{
-		// padding
-		memset(tcp_option_buffer_c + tcp_option_length_c, 1, 2);
-		tcp_option_length_c += 2;
-		memset(tcp_option_buffer_s + tcp_option_length_s, 1, 2);
-		tcp_option_length_s += 2;
-
-		struct tcp_option_sack *option_c = (struct tcp_option_sack *)(tcp_option_buffer_c + tcp_option_length_c);
-		option_c->kind = 4;
-		option_c->length = 2;
-		tcp_option_length_c += sizeof(struct tcp_option_sack);
-
-		struct tcp_option_sack *option_s = (struct tcp_option_sack *)(tcp_option_buffer_s + tcp_option_length_s);
-		option_s->kind = 4;
-		option_s->length = 2;
-		tcp_option_length_s += sizeof(struct tcp_option_sack);
-	}
-
-	/*
-	 * Time Stamp option: Kind: 8, Length: 10
-	 * +---------+---------+-----+-----+
-	 * | Kind=8  |Length=10|tsval|tsecr|
-	 * +---------+---------+-----+-----+
-	 *      1         1       4     4
-	 */
-	if (client->timestamp_perm && server->timestamp_perm)
-	{
-		// padding
-		memset(tcp_option_buffer_c + tcp_option_length_c, 1, 2);
-		tcp_option_length_c += 2;
-		memset(tcp_option_buffer_s + tcp_option_length_s, 1, 2);
-		tcp_option_length_s += 2;
-		memset(tcp_option_buffer_c2 + tcp_option_length_c2, 1, 2);
-		tcp_option_length_c2 += 2;
-
-		struct tcp_option_time_stamp *option_c = (struct tcp_option_time_stamp *)(tcp_option_buffer_c + tcp_option_length_c);
-		option_c->kind = 8;
-		option_c->length = 10;
-		option_c->tsval = htonl(client->ts_val);
-		option_c->tsecr = htonl(0);
-		tcp_option_length_c += sizeof(struct tcp_option_time_stamp);
-
-		struct tcp_option_time_stamp *option_s = (struct tcp_option_time_stamp *)(tcp_option_buffer_s + tcp_option_length_s);
-		option_s->kind = 8;
-		option_s->length = 10;
-		option_s->tsval = htonl(server->ts_val);
-		option_s->tsecr = htonl(client->ts_val);
-		tcp_option_length_s += sizeof(struct tcp_option_time_stamp);
-
-		struct tcp_option_time_stamp *option_c2 = (struct tcp_option_time_stamp *)(tcp_option_buffer_c2 + tcp_option_length_c2);
-		option_c2->kind = 8;
-		option_c2->length = 10;
-		option_c2->tsval = htonl(client->ts_val);
-		option_c2->tsecr = htonl(server->ts_val);
-		tcp_option_length_c2 += sizeof(struct tcp_option_time_stamp);
-	}
-
-	if (client->addr.ss_family == AF_INET6)
-	{
-		struct sockaddr_in6 *sk_client = (struct sockaddr_in6 *)&client->addr;
-		struct sockaddr_in6 *sk_server = (struct sockaddr_in6 *)&server->addr;
-		uint16_t port_client = sk_client->sin6_port;
-		uint16_t port_server = sk_server->sin6_port;
-
-		// C -> S
-		length = tcp_packet_v6_construct(
-       		buffer, // buffer
-       	 	&raw_socket_c->mac_addr, &raw_socket_s->mac_addr, 0, ETH_P_IPV6, // Ether
-        	&sk_client->sin6_addr, &sk_server->sin6_addr, 55, // IPv6
-			port_client, port_server, c_seq, 0, TCP_SYN_FLAG, client->window, // TCP Header
-			tcp_option_buffer_c, tcp_option_length_c, // TCP Options
-			NULL, 0); // Payload
-		raw_socket_send(raw_socket_c, buffer, length);
-		c_seq += 1;
-
-		// S -> C
-		length = tcp_packet_v6_construct(
-       		buffer, // buffer
-       	 	&raw_socket_s->mac_addr, &raw_socket_c->mac_addr, 0, ETH_P_IPV6, // Ether
-        	&sk_server->sin6_addr, &sk_client->sin6_addr,  65, // IPv6
-			port_server, port_client, s_seq, c_seq, TCP_SYN_FLAG | TCP_ACK_FLAG, server->window, // TCP Header
-			tcp_option_buffer_s, tcp_option_length_s, // TCP Options
-			NULL, 0); // Payload
-		raw_socket_send(raw_socket_s, buffer, length);
-		s_seq += 1;
-
-		// C -> S
-		length = tcp_packet_v6_construct(
-       		buffer, // buffer
-       	 	&raw_socket_c->mac_addr, &raw_socket_s->mac_addr, 0, ETH_P_IPV6, // Ether
-        	&sk_client->sin6_addr, &sk_server->sin6_addr, 55,  // IPv6
-			port_client, port_server, c_seq, s_seq, TCP_ACK_FLAG, client->window, // TCP Header
-			tcp_option_buffer_c2, tcp_option_length_c2, // TCP Options
-			NULL, 0); // Payload
-		raw_socket_send(raw_socket_c, buffer, length);
-	}
-	else
-	{
-		struct sockaddr_in *sk_client = (struct sockaddr_in *)&client->addr;
-		struct sockaddr_in *sk_server = (struct sockaddr_in *)&server->addr;
-		uint16_t port_client = sk_client->sin_port;
-		uint16_t port_server = sk_server->sin_port;
-
-		// C -> S
-		length = tcp_packet_v4_construct(
-			buffer,                                                               // buffer
-			&raw_socket_c->mac_addr, &raw_socket_s->mac_addr, 0, ETH_P_IP,        // Ether
-			&sk_client->sin_addr, &sk_server->sin_addr, 0, 55, 0x11,              // IPv4
-			port_client, port_server, c_seq, 0, TCP_SYN_FLAG, client->window, // TCP Header
-			tcp_option_buffer_c, tcp_option_length_c, // TCP Options
-			NULL, 0);
-		raw_socket_send(raw_socket_c, buffer, length);
-		c_seq += 1;
-
-		// S -> C
-		length = tcp_packet_v4_construct(
-			buffer,                                                                              // buffer
-			&raw_socket_s->mac_addr, &raw_socket_c->mac_addr, 0, ETH_P_IP,                       // Ether
-			&sk_server->sin_addr,&sk_client->sin_addr, 0, 65, 0x12,                              // IPv4
-			port_server, port_client, s_seq, c_seq, TCP_SYN_FLAG | TCP_ACK_FLAG, server->window, // TCP Header
-			tcp_option_buffer_s, tcp_option_length_s, // TCP Options
-			NULL, 0);
-		raw_socket_send(raw_socket_s, buffer, length);
-		s_seq += 1;
-
-		// C -> S
-		length = tcp_packet_v4_construct(
-			buffer, // buffer
-			&raw_socket_c->mac_addr, &raw_socket_s->mac_addr, 0, ETH_P_IP, // Ether
-			&sk_client->sin_addr, &sk_server->sin_addr, 0, 55, 0x13, // IPv4
-			port_client, port_server, c_seq, s_seq, TCP_ACK_FLAG, client->window, // TCP Header
-			tcp_option_buffer_c2, tcp_option_length_c2, // TCP Options
-			NULL, 0);
-		raw_socket_send(raw_socket_c, buffer, length);
-	}
-
-    raw_socket_destory(raw_socket_c);
-    raw_socket_destory(raw_socket_s);
-    
-	return 0;
-}
-
 static int overwrite_tcp_mss(struct tfe_cmsg *cmsg, struct tcp_restore_info *restore)
 {
 	int ret = 0;
@@ -588,8 +336,6 @@ static int payload_handler_cb(struct nfq_q_handle *qh, struct nfgenmsg *nfmsg, s
 	int ret = 0;
 	int fd_downstream = 0;
 	int fd_upstream = 0;
-	int fd_fake_c = 0;
-	int fd_fake_s = 0;
 	int hit_tcpopt = 0;
 	uint16_t cmsg_offset = 0;
 	uint8_t restore_opt_len = 0;
@@ -603,7 +349,6 @@ static int payload_handler_cb(struct nfq_q_handle *qh, struct nfgenmsg *nfmsg, s
 	struct pkt_info pktinfo;
 	struct tcp_restore_info restore_info;
 	uint8_t stream_protocol_in_char = 0;
-	uint8_t enalbe_decrypted_traffic_steering = 0;
 	uint16_t size = 0;
 	// uint64_t chaining_rule_id = 0; // only use for acceptv4
 	struct acceptor_kni_v3 *__ctx = (struct acceptor_kni_v3 *)data;
@@ -746,33 +491,7 @@ static int payload_handler_cb(struct nfq_q_handle *qh, struct nfgenmsg *nfmsg, s
 	tfe_cmsg_get_value(cmsg, TFE_CMSG_TCP_RESTORE_PROTOCOL, (unsigned char *)&stream_protocol_in_char, sizeof(stream_protocol_in_char), &size);
 	// tfe_cmsg_get_value(cmsg, TFE_CMSG_TCP_DECRYPTED_TRAFFIC_STEERING, (unsigned char *)&enalbe_decrypted_traffic_steering, sizeof(enalbe_decrypted_traffic_steering), &size);
 
-	if (steering_device_is_available() && (
-			(STREAM_PROTO_PLAIN == (enum tfe_stream_proto)stream_protocol_in_char && __ctx->proxy->traffic_steering_options.enable_steering_http) ||
-		 	(STREAM_PROTO_SSL == (enum tfe_stream_proto)stream_protocol_in_char && __ctx->proxy->traffic_steering_options.enable_steering_ssl) ||
-			enalbe_decrypted_traffic_steering == 1))
-	{
-		if (fake_tcp_handshake(__ctx->proxy, &restore_info) == -1)
-		{
-			TFE_LOG_ERROR(g_default_logger, "Failed at fake_tcp_handshake()");
-			goto end;
-		}
-
-		fd_fake_c = tfe_tcp_restore_fd_create(&(restore_info.client), &(restore_info.server), __ctx->proxy->traffic_steering_options.device_client, __ctx->proxy->traffic_steering_options.so_mask_client);
-		if (fd_fake_c < 0)
-		{
-			TFE_LOG_ERROR(g_default_logger, "Failed at tcp_restore_fd_create(fd_fake_c)");
-			goto end;
-		}
-
-		fd_fake_s = tfe_tcp_restore_fd_create(&(restore_info.server), &(restore_info.client), __ctx->proxy->traffic_steering_options.device_server, __ctx->proxy->traffic_steering_options.so_mask_server);
-		if (fd_fake_s < 0)
-		{
-			TFE_LOG_ERROR(g_default_logger, "Failed at tcp_restore_fd_create(fd_fake_s)");
-			goto end;
-		}
-	}
-
-	if (tfe_proxy_fds_accept(__ctx->proxy, fd_downstream, fd_upstream, fd_fake_c, fd_fake_s, cmsg) < 0)
+	if (tfe_proxy_fds_accept(__ctx->proxy, fd_downstream, fd_upstream, 0, 0, cmsg) < 0)
 	{
 		TFE_LOG_ERROR(g_default_logger, "Failed at tfe_proxy_fds_accept()");
 		goto end;
@@ -800,14 +519,6 @@ static int payload_handler_cb(struct nfq_q_handle *qh, struct nfgenmsg *nfmsg, s
 	return nfq_set_verdict(qh, id, NF_ACCEPT, pktinfo.ip_totlen, raw_payload);
 
 end:
-	if (fd_fake_c > 0)
-	{
-		close(fd_fake_c);
-	}
-	if (fd_fake_s > 0)
-	{
-		close(fd_fake_s);
-	}
 	if (fd_upstream > 0)
 	{
 		TFE_PROXY_STAT_INCREASE(STAT_FD_CLOSE_BY_KNI_ACCEPT_FAIL, 1);
diff --git a/platform/src/acceptor_kni_v4.cpp b/platform/src/acceptor_kni_v4.cpp
index f054262..941a084 100644
--- a/platform/src/acceptor_kni_v4.cpp
+++ b/platform/src/acceptor_kni_v4.cpp
@@ -126,19 +126,29 @@ static void *worker_thread_cycle(void *arg)
 {
     struct packet_io_thread_ctx *thread_ctx = (struct packet_io_thread_ctx *)arg;
     struct packet_io *handle = thread_ctx->ref_io;
-    void * logger = thread_ctx->logger;
+    void *logger = thread_ctx->logger;
 
+#define MAX_REBUFF_SIZE 2048
+    char buffer[MAX_REBUFF_SIZE];
     int pkg_len = 0;
     char thread_name[16];
-    int n_pkt_recv_from_nf = 0;
-    int n_pkt_recv_from_tap = 0;
-    int n_pkt_recv_from_tap_c = 0;
-    int n_pkt_recv_from_tap_s = 0;
+    int n_pkt_recv = 0;
+    int thread_index = thread_ctx->thread_index;
+    int using_iouring_mode = is_enable_iouring(handle);
 
-    snprintf(thread_name, sizeof(thread_name), "pkt:worker-%d", thread_ctx->thread_index);
+    int fd_on_tap_0 = thread_ctx->tap_ctx->tap_fd;
+    int fd_on_tap_c = thread_ctx->tap_ctx->tap_c;
+    int fd_on_tap_s = thread_ctx->tap_ctx->tap_s;
+
+    struct io_uring_instance *io_uring_on_tap_0 = thread_ctx->tap_ctx->io_uring_fd;
+    struct io_uring_instance *io_uring_on_tap_c = thread_ctx->tap_ctx->io_uring_c;
+    struct io_uring_instance *io_uring_on_tap_s = thread_ctx->tap_ctx->io_uring_s;
+
+    snprintf(thread_name, sizeof(thread_name), "pkt:worker-%d", thread_index);
     prctl(PR_SET_NAME, (unsigned long long)thread_name, NULL, NULL, NULL);
 
-    while (!worker_thread_ready) {
+    while (!worker_thread_ready)
+    {
         sleep(1);
     }
 
@@ -147,56 +157,59 @@ static void *worker_thread_cycle(void *arg)
         goto error_out;
     }
 
-    if (is_enable_iouring(handle)) {
-        io_uring_register_read_callback(thread_ctx->tap_ctx->io_uring_fd, handle_raw_packet_from_tap, thread_ctx);
-        io_uring_register_read_callback(thread_ctx->tap_ctx->io_uring_c, handle_decryption_packet_from_tap, thread_ctx);
-        io_uring_register_read_callback(thread_ctx->tap_ctx->io_uring_s, handle_decryption_packet_from_tap, thread_ctx);
-    }
-    else {
-        thread_ctx->tap_ctx->buff_size = 3000;
-        thread_ctx->tap_ctx->buff = ALLOC(char, thread_ctx->tap_ctx->buff_size);
+    if (using_iouring_mode)
+    {
+        io_uring_set_read_cb(io_uring_on_tap_0, handle_raw_packet_from_tap, thread_ctx);
+        io_uring_set_read_cb(io_uring_on_tap_c, handle_decryption_packet_from_tap, thread_ctx);
+        io_uring_set_read_cb(io_uring_on_tap_s, handle_decryption_packet_from_tap, thread_ctx);
     }
 
-    TFE_LOG_INFO(logger, "%s: worker thread %d is running", "LOG_TAG_KNI", thread_ctx->thread_index);
+    TFE_LOG_INFO(logger, "%s: worker thread %d is running", "LOG_TAG_KNI", thread_index);
 
-    while(1) {
-        n_pkt_recv_from_nf = packet_io_polling_nf_interface(handle, thread_ctx->thread_index, thread_ctx);
-        if (is_enable_iouring(handle)) {
-            n_pkt_recv_from_tap = io_uring_peek_ready_entrys(thread_ctx->tap_ctx->io_uring_fd);
-            n_pkt_recv_from_tap_c = io_uring_peek_ready_entrys(thread_ctx->tap_ctx->io_uring_c);
-            n_pkt_recv_from_tap_s = io_uring_peek_ready_entrys(thread_ctx->tap_ctx->io_uring_s);
+    while (1)
+    {
+        n_pkt_recv = packet_io_polling_nf_interface(handle, thread_index, thread_ctx);
+        if (using_iouring_mode)
+        {
+            n_pkt_recv += io_uring_polling(io_uring_on_tap_0);
+            n_pkt_recv += io_uring_polling(io_uring_on_tap_c);
+            n_pkt_recv += io_uring_polling(io_uring_on_tap_s);
         }
-        else {
-            if ((pkg_len = tap_read(thread_ctx->tap_ctx->tap_fd, thread_ctx->tap_ctx->buff, thread_ctx->tap_ctx->buff_size, logger)) > 0)
+        else
+        {
+            if ((pkg_len = tap_read(fd_on_tap_0, buffer, MAX_REBUFF_SIZE, logger)) > 0)
             {
-                handle_raw_packet_from_tap(thread_ctx->tap_ctx->buff, pkg_len, thread_ctx);
+                n_pkt_recv++;
+                handle_raw_packet_from_tap(buffer, pkg_len, thread_ctx);
             }
 
-            if ((pkg_len = tap_read(thread_ctx->tap_ctx->tap_c, thread_ctx->tap_ctx->buff, thread_ctx->tap_ctx->buff_size, logger)) > 0)
+            if ((pkg_len = tap_read(fd_on_tap_c, buffer, MAX_REBUFF_SIZE, logger)) > 0)
             {
-                handle_decryption_packet_from_tap(thread_ctx->tap_ctx->buff, pkg_len, thread_ctx);
+                n_pkt_recv++;
+                handle_decryption_packet_from_tap(buffer, pkg_len, thread_ctx);
             }
 
-            if ((pkg_len = tap_read(thread_ctx->tap_ctx->tap_s, thread_ctx->tap_ctx->buff, thread_ctx->tap_ctx->buff_size, logger)) > 0)
+            if ((pkg_len = tap_read(fd_on_tap_s, buffer, MAX_REBUFF_SIZE, logger)) > 0)
             {
-                handle_decryption_packet_from_tap(thread_ctx->tap_ctx->buff, pkg_len, thread_ctx);
+                n_pkt_recv++;
+                handle_decryption_packet_from_tap(buffer, pkg_len, thread_ctx);
             }
         }
 
-        if (n_pkt_recv_from_nf == 0 && n_pkt_recv_from_tap == 0 && n_pkt_recv_from_tap_c == 0 && n_pkt_recv_from_tap_s == 0)
+        if (n_pkt_recv == 0)
         {
             packet_io_thread_wait(handle, thread_ctx, -1);
         }
 
-        if (__atomic_fetch_add(&thread_ctx->session_table_need_reset, 0, __ATOMIC_RELAXED) > 0)
+        if (ATOMIC_READ(&thread_ctx->session_table_need_reset) > 0)
         {
             session_table_reset(thread_ctx->session_table);
-            __atomic_fetch_and(&thread_ctx->session_table_need_reset, 0, __ATOMIC_RELAXED);
+            ATOMIC_ZERO(&thread_ctx->session_table_need_reset);
         }
     }
 
 error_out:
-    TFE_LOG_ERROR(logger, "%s: worker thread %d exiting", LOG_TAG_SCE, thread_ctx->thread_index);
+    TFE_LOG_ERROR(logger, "%s: worker thread %d exiting", LOG_TAG_SCE, thread_index);
     return (void *)NULL;
 }
 
diff --git a/platform/src/ssl_stream.cpp b/platform/src/ssl_stream.cpp
index 60db7cb..46fb62a 100644
--- a/platform/src/ssl_stream.cpp
+++ b/platform/src/ssl_stream.cpp
@@ -1320,7 +1320,7 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events,
 		jiffies_ms=(ctx->end.tv_sec-ctx->start.tv_sec)*1000+(ctx->end.tv_nsec-ctx->start.tv_nsec)/1000000;
 		if(jiffies_ms>LATENCY_WARNING_THRESHOLD_MS)
 		{
-			TFE_LOG_ERROR(mgr->logger, "Warning: ssl connect server latency %ld ms: addr=%s, sni=%s",
+			TFE_LOG_INFO(mgr->logger, "Warning: ssl connect server latency %ld ms: addr=%s, sni=%s",
 									jiffies_ms,
 									s_stream->tcp_stream->str_stream_info,
 									s_upstream->client_hello->sni);
@@ -1948,7 +1948,7 @@ static void ssl_client_connected_eventcb(struct bufferevent * bev, short events,
 		jiffies_ms=(ctx->end.tv_sec-ctx->start.tv_sec)*1000+(ctx->end.tv_nsec-ctx->start.tv_nsec)/1000000;
 		if(jiffies_ms>LATENCY_WARNING_THRESHOLD_MS)
 		{
-			TFE_LOG_ERROR(mgr->logger, "Warning: ssl connect client latency %ld ms: addr=%s, sni=%s",
+			TFE_LOG_INFO(mgr->logger, "Warning: ssl connect client latency %ld ms: addr=%s, sni=%s",
 										jiffies_ms,
 										s_stream->tcp_stream->str_stream_info,
 										s_upstream->client_hello->sni);
diff --git a/platform/src/tcp_stream.cpp b/platform/src/tcp_stream.cpp
index be895a0..439908b 100644
--- a/platform/src/tcp_stream.cpp
+++ b/platform/src/tcp_stream.cpp
@@ -1603,7 +1603,7 @@ static void get_tcp_option_from_cmsg(struct tfe_cmsg *cmsg, struct tfe_tcp_optio
 	}
 }
 
-void __stream_fd_option_setup(struct tfe_stream_private * _stream, evutil_socket_t fd, tfe_conn_dir dir)
+void __stream_fd_option_setup(struct tfe_stream_private * _stream, evutil_socket_t fd, tfe_conn_dir dir, int overwrite_ttl)
 {
 	struct tfe_stream * stream = &_stream->head;
 	struct tfe_proxy_tcp_options * tcp_options = &_stream->proxy_ref->tcp_options;
@@ -1720,21 +1720,31 @@ void __stream_fd_option_setup(struct tfe_stream_private * _stream, evutil_socket
         errno = 0;
     }
 
-	if (options.tcp_ttl > 0)
+	if (overwrite_ttl > 0)
 	{
-		if (__fd_ttl_option_setup(_stream, fd, options.tcp_ttl) < 0)
+		if (__fd_ttl_option_setup(_stream, fd, overwrite_ttl) < 0)
 		{
 			TFE_LOG_ERROR(g_default_logger, "%s: Failed at setup FD's ttl option, ttl = %d, fd = %d",
-				stream->str_stream_info, options.tcp_ttl, fd);
+						  stream->str_stream_info, overwrite_ttl, fd);
+		}
+	}
+	else
+	{
+		if (options.tcp_ttl > 0)
+		{
+			if (__fd_ttl_option_setup(_stream, fd, options.tcp_ttl) < 0)
+			{
+				TFE_LOG_ERROR(g_default_logger, "%s: Failed at setup FD's ttl option, ttl = %d, fd = %d",
+							  stream->str_stream_info, options.tcp_ttl, fd);
+			}
 		}
 	}
 
 	TFE_LOG_DEBUG(g_default_logger,
-		"%p %s %s: fetch tcp options, nodelay: %d ttl: %d keepalive: %d keepcnt: %d keepidle: %d keepintvl: %d user_timeout: %d",
-		stream, stream->str_stream_info, (dir == CONN_DIR_DOWNSTREAM ? "downstream" : "upstream"),
-		options.tcp_nodelay, options.tcp_ttl, options.tcp_keepalive,
-		options.tcp_keepcnt, options.tcp_keepidle, options.tcp_keepintvl, options.tcp_user_timeout);
-
+				  "%p %s %s: fetch tcp options, nodelay: %d ttl: %d keepalive: %d keepcnt: %d keepidle: %d keepintvl: %d user_timeout: %d",
+				  stream, stream->str_stream_info, (dir == CONN_DIR_DOWNSTREAM ? "downstream" : "upstream"),
+				  options.tcp_nodelay, overwrite_ttl > 0 ? overwrite_ttl : options.tcp_ttl, options.tcp_keepalive,
+				  options.tcp_keepcnt, options.tcp_keepidle, options.tcp_keepintvl, options.tcp_user_timeout);
 }
 
 int tfe_stream_init_by_fds(struct tfe_stream * stream, evutil_socket_t fd_downstream, evutil_socket_t fd_upstream, evutil_socket_t fd_fake_c,  evutil_socket_t fd_fake_s)
@@ -1763,13 +1773,13 @@ int tfe_stream_init_by_fds(struct tfe_stream * stream, evutil_socket_t fd_downst
 	_stream->str_stream_addr = tfe_stream_addr_to_str(_stream->head.addr);
 	stream->str_stream_info = _stream->str_stream_addr;
 	
-	__stream_fd_option_setup(_stream, fd_downstream, CONN_DIR_DOWNSTREAM);
-	__stream_fd_option_setup(_stream, fd_upstream, CONN_DIR_UPSTREAM);
+	__stream_fd_option_setup(_stream, fd_downstream, CONN_DIR_DOWNSTREAM, 0);
+	__stream_fd_option_setup(_stream, fd_upstream, CONN_DIR_UPSTREAM, 0);
 
 	if (_stream->is_decrypted_traffic_steering)
 	{
-		__stream_fd_option_setup(_stream, fd_fake_s, CONN_DIR_UPSTREAM);
-		__stream_fd_option_setup(_stream, fd_fake_c, CONN_DIR_DOWNSTREAM);
+		__stream_fd_option_setup(_stream, fd_fake_s, CONN_DIR_UPSTREAM, TFE_FAKE_S_DEFAULT_TTL);
+		__stream_fd_option_setup(_stream, fd_fake_c, CONN_DIR_DOWNSTREAM, TFE_FAKE_C_DEFAULT_TTL);
 
 		_stream->conn_fake_s = __conn_private_create_by_fake_fd(_stream, fd_fake_s);
 		if (_stream->conn_fake_s == NULL)