TSG-2993 将 TFE key keeper 哈希表的 pthread_mutex 改为 pthread_rwlock

	* 插入新证书时若哈希表满，则删除 10% 最老的证书
	* 查询到结果之后再判断是否超时，若超时则删除，并重新查询 certstore

2 files changed, 106 insertions, 39 deletions

diff --git a/platform/src/key_keeper.cpp b/platform/src/key_keeper.cpp
index fa932df..a48854b 100644
--- a/platform/src/key_keeper.cpp
+++ b/platform/src/key_keeper.cpp
@@ -23,6 +23,7 @@
 #define HTABLE_MAX_KEY_LEN 256
 #define KEYRING_EXSITED 0
 #define KEYRING_NOT_EXSITED -1
+#define KEYRING_EXPIRE -2
 
 enum key_keeper_mode{
 	KK_MODE_CERT_STORE = 0,
@@ -37,7 +38,9 @@ struct key_keeper
 	char cert_store_host[TFE_SYMBOL_MAX];
 	unsigned int cert_store_port;
 	unsigned int hash_slot_size;
-	unsigned int hash_expire_seconds;
+    unsigned int hash_expire_seconds;
+    unsigned int del_elem_num_once_when_full;
+    unsigned int max_elem_num;
     MESA_htable_handle cert_cache;
     void* logger;
 	X509* trusted_ca_cert;
@@ -51,6 +54,7 @@ struct key_keeper
 	int cert_expire_time;
 	unsigned int enable_health_check;
 	pthread_t thread;
+    pthread_rwlock_t rwlock;
 };
 
 
@@ -60,6 +64,7 @@ struct keyring_private
 	struct keyring head;
 	pthread_mutex_t mutex;
 	size_t references;
+    time_t update_time;
 };
 
 struct key_keeper_promise_ctx
@@ -335,9 +340,16 @@ static long keyring_local_cache_query_cb(void * data, const uchar * key, uint si
 	}
 	else
 	{
-		struct promise* p = (struct promise*)user_arg;
-		promise_success(p, data);
-		return KEYRING_EXSITED;
+        struct promise *p = (struct promise *)user_arg;
+        if (time(NULL) - kyr->update_time > ((struct key_keeper_promise_ctx *)promise_get_ctx(p))->ref_keeper->hash_expire_seconds)
+        {
+            return KEYRING_EXPIRE;
+        }
+		else
+		{
+			promise_success(p, data);
+			return KEYRING_EXSITED;
+		}
 	}
 }
 
@@ -389,12 +401,24 @@ static void certstore_rpc_on_succ(void* result, void* user)
 		if(!ctx->ref_keeper->no_cache)
 		{
 			keyring_ref_inc(kyr);
-			int ret = MESA_htable_add(htable, key, key_len, (void*)kyr);
-			if(ret<0)
+            kyr->update_time = time(NULL);
+            pthread_rwlock_wrlock(&(ctx->ref_keeper->rwlock));
+            if (MESA_htable_get_elem_num(ctx->ref_keeper->cert_cache) == ctx->ref_keeper->max_elem_num)
+            {
+                MESA_htable_del_oldest_manual(htable, NULL, ctx->ref_keeper->del_elem_num_once_when_full);
+                TFE_LOG_DEBUG(ctx->ref_keeper->logger, "Key keeper cache full: %d del: %d left: %d", ctx->ref_keeper->max_elem_num, ctx->ref_keeper->del_elem_num_once_when_full, MESA_htable_get_elem_num(ctx->ref_keeper->cert_cache));
+            }
+            int ret = MESA_htable_add(htable, key, key_len, (void*)kyr);
+            pthread_rwlock_unlock(&(ctx->ref_keeper->rwlock));
+            if(ret<0)
 			{
 				key_keeper_free_keyring((struct keyring*)kyr);
 			}
-		}
+            else
+            {
+                TFE_LOG_DEBUG(ctx->ref_keeper->logger, "Key keeper cache add key: %s", ctx->key);
+            }
+        }
 		ctx->ref_keeper->stat.new_issue++;
 		promise_success(p, (void*)kyr);
 		key_keeper_free_keyring((struct keyring*)kyr);
@@ -425,17 +449,23 @@ static void key_keeper_free_serialized(void* data)
 	key_keeper_free_keyring(&(kyr->head));
 }
 
-static MESA_htable_handle create_hash_table(unsigned int slot_size, unsigned int expire_seconds)
+static MESA_htable_handle create_hash_table(unsigned int slot_size, unsigned int max_elem_num)
 {
 	UNUSED	int ret = 0;
-	unsigned max_num = slot_size * 4;
 	MESA_htable_handle htable = MESA_htable_born();
 	ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_SCREEN_PRINT_CTRL, 0);
-	ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_THREAD_SAFE, 1);
-	ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_MUTEX_NUM, 16);
-	ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_SLOT_SIZE, slot_size);
-	ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_MAX_ELEMENT_NUM, max_num);
-	ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_EXPIRE_TIME, expire_seconds);
+    /*
+	 * Use rwlock instead of htable mutex:
+	 * MHO_THREAD_SAFE must be set to 0,
+	 * MHO_MUTEX_NUM   must be set to 1,
+	 * MHO_EXPIRE_TIME must be set to 0,
+	 * otherwise MESA_htable_del_oldest_manual() execution error
+	 */
+    ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_THREAD_SAFE, 0);
+    ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_MUTEX_NUM, 1);
+    ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_SLOT_SIZE, slot_size);
+    ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_HASH_MAX_ELEMENT_NUM, max_elem_num);
+    ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_EXPIRE_TIME, 0);
 	ret = __wrapper_MESA_htable_set_opt_int(htable, MHO_ELIMIMINATE_TYPE,
 		HASH_ELIMINATE_ALGO_FIFO);
 	ret = __wrapper_MESA_htable_set_opt_func(htable, MHO_CBFUN_DATA_FREE,
@@ -449,7 +479,11 @@ static MESA_htable_handle create_hash_table(unsigned int slot_size, unsigned int
 
 void key_keeper_destroy(struct key_keeper *keeper)
 {
-	MESA_htable_destroy(keeper->cert_cache, NULL);
+    if (!keeper->no_cache)
+	{
+    	pthread_rwlock_destroy(&(keeper->rwlock));
+    	MESA_htable_destroy(keeper->cert_cache, NULL);
+	}
 	X509_free(keeper->trusted_ca_cert);
 	EVP_PKEY_free(keeper->trusted_ca_key);
 
@@ -573,16 +607,22 @@ struct key_keeper* key_keeper_init(const char * profile, const char* section, vo
 
 	MESA_load_profile_uint_def(profile, section, "enable_health_check", &(keeper->enable_health_check), 1);
 	MESA_load_profile_uint_def(profile, section, "cert_store_port", &(keeper->cert_store_port), 80);
-	MESA_load_profile_uint_def(profile, section, "hash_slot_size", &(keeper->hash_slot_size), 1024*128);
-	MESA_load_profile_uint_def(profile, section, "hash_expire_seconds", &(keeper->hash_expire_seconds), 5*60);
-	MESA_load_profile_uint_def(profile, section, "no_cache", &(keeper->no_cache), 0);
+	MESA_load_profile_int_def(profile, section, "cert_expire_time", &(keeper->cert_expire_time), 24);	
+    MESA_load_profile_uint_def(profile, section, "no_cache", &(keeper->no_cache), 0);
+    if (!keeper->no_cache)
+    {
+        MESA_load_profile_uint_def(profile, section, "hash_expire_seconds", &(keeper->hash_expire_seconds), 5 * 60);
+        MESA_load_profile_uint_def(profile, section, "hash_slot_size", &(keeper->hash_slot_size), 1024 * 128);
+        if (keeper->cert_expire_time != -1)
+        {
+            keeper->hash_expire_seconds = MIN(keeper->cert_expire_time * 1800, (int)(keeper->hash_expire_seconds));
+        }
 
-	MESA_load_profile_int_def(profile, section, "cert_expire_time", &(keeper->cert_expire_time), 24);
-	keeper->cert_cache = create_hash_table(keeper->hash_slot_size, keeper->hash_expire_seconds);
-	if (keeper->cert_expire_time != -1)
-	{
-		keeper->hash_expire_seconds = MIN(keeper->cert_expire_time * 1800, (int)(keeper->hash_expire_seconds));
-	}	
+        keeper->max_elem_num = keeper->hash_slot_size * 4;
+        keeper->del_elem_num_once_when_full = (keeper->max_elem_num / 10) > 0 ? (keeper->max_elem_num / 10) : 1;
+        keeper->cert_cache = create_hash_table(keeper->hash_slot_size, keeper->max_elem_num);
+        pthread_rwlock_init(&(keeper->rwlock), NULL);
+    }
 	if(0==strcmp(keeper->untrusted_ca_path, keeper->trusted_ca_path))
 	{
 		TFE_LOG_ERROR(logger, "Warnning: Trusted and Untrusted Root CA share the same path %s .", keeper->trusted_ca_path);
@@ -619,11 +659,11 @@ struct key_keeper* key_keeper_init(const char * profile, const char* section, vo
 			goto error_out;
 		}
 	}
-	
-	TFE_LOG_INFO(logger, "MESA_load_profile, [%s]: mode:%s, no_cache:%u ,ca_path:%s, untrusted_ca_path:%s, cert_store_host:%s, cert_store_port:%d, hash_slot_size:%d, hash_expire_seconds:%d, cert_expire_time:%d",
-		section, tmp, keeper->no_cache, keeper->trusted_ca_path, keeper->untrusted_ca_path, keeper->cert_store_host, keeper->cert_store_port, keeper->hash_slot_size, keeper->hash_expire_seconds, keeper->cert_expire_time);
 
-	return keeper;
+    TFE_LOG_INFO(logger, "Key keeper cache, mode:%s, no_cache:%u, ca_path:%s, untrusted_ca_path:%s, cert_store_host:%s, cert_store_port:%d, hash_slot_size:%d, hash_expire_seconds:%d, cert_expire_time:%d, max_elem_num:%d, del_elem_num_once_when_full:%d",
+                 tmp, keeper->no_cache, keeper->trusted_ca_path, keeper->untrusted_ca_path, keeper->cert_store_host, keeper->cert_store_port, keeper->hash_slot_size, keeper->hash_expire_seconds, keeper->cert_expire_time, keeper->max_elem_num, keeper->del_elem_num_once_when_full);
+
+    return keeper;
 
 error_out:
 	key_keeper_destroy(keeper);
@@ -696,13 +736,24 @@ void key_keeper_async_ask(struct future * f, struct key_keeper * keeper, const c
 	keeper->stat.ask_times++;
 	if(!keeper->no_cache)
 	{
-		MESA_htable_search_cb(keeper->cert_cache, (const unsigned char*)(ctx->key), ctx->key_len, keyring_local_cache_query_cb, p, &cb_rtn);
-		if(cb_rtn == KEYRING_EXSITED)
+        char *tmp = tfe_strdup((const char *)ctx->key);
+        pthread_rwlock_rdlock(&(keeper->rwlock));
+        MESA_htable_search_cb(keeper->cert_cache, (const unsigned char*)(ctx->key), ctx->key_len, keyring_local_cache_query_cb, p, &cb_rtn);
+        pthread_rwlock_unlock(&(keeper->rwlock));
+        TFE_LOG_DEBUG(keeper->logger, "Key keeper cache search key: %s, found: %d (0:KEYRING_EXSITED, -1:KEYRING_NOT_EXSITED, -2:KEYRING_EXPIRE)", tmp, cb_rtn);
+        free(tmp);
+        if(cb_rtn == KEYRING_EXSITED)
 		{
 			//printf("KEYRING_EXSITED\n");
 			return;
 		}
-	}
+        if (cb_rtn == KEYRING_EXPIRE)
+		{
+            pthread_rwlock_wrlock(&(keeper->rwlock));
+            MESA_htable_del(keeper->cert_cache, (const unsigned char *)(ctx->key), ctx->key_len, NULL);
+			pthread_rwlock_unlock(&(keeper->rwlock));
+        }
+    }
 	switch(keeper->work_mode)
 	{
 		case KK_MODE_CERT_STORE:
@@ -749,11 +800,23 @@ void key_keeper_async_ask(struct future * f, struct key_keeper * keeper, const c
 				if(!keeper->no_cache)
 				{
 					keyring_ref_inc(kyr);
-					int ret = MESA_htable_add(ctx->ref_keeper->cert_cache, ctx->key, ctx->key_len, (void*)kyr);
-					if(ret < 0)
+                    kyr->update_time = time(NULL);
+                    pthread_rwlock_wrlock(&(keeper->rwlock));
+                    if (MESA_htable_get_elem_num(keeper->cert_cache) == keeper->max_elem_num)
+                    {
+                        MESA_htable_del_oldest_manual(keeper->cert_cache, NULL, keeper->del_elem_num_once_when_full);
+                        TFE_LOG_DEBUG(keeper->logger, "Key keeper cache full: %d del: %d left: %d", keeper->max_elem_num, keeper->del_elem_num_once_when_full, MESA_htable_get_elem_num(keeper->cert_cache));
+                    }
+                    int ret = MESA_htable_add(ctx->ref_keeper->cert_cache, ctx->key, ctx->key_len, (void *)kyr);
+                    pthread_rwlock_unlock(&(keeper->rwlock));
+                    if(ret < 0)
 					{
 						key_keeper_free_keyring((struct keyring*)kyr);
 					}
+					else
+					{
+                        TFE_LOG_DEBUG(keeper->logger, "Key keeper cache add key: %s", ctx->key);
+                    }		
 				}
 				promise_success(p, (void*)kyr);
 				keeper->stat.new_issue++;
@@ -770,8 +833,12 @@ void key_keeper_async_ask(struct future * f, struct key_keeper * keeper, const c
 }
 void key_keeper_statistic(struct key_keeper *keeper, struct key_keeper_stat* result)
 {
-	keeper->stat.cached_num=MESA_htable_get_elem_num(keeper->cert_cache);
-	*result=keeper->stat;
-	return;
-}
-
+	if (!keeper->no_cache)
+	{
+    	// pthread_rwlock_rdlock(&(keeper->rwlock));
+    	keeper->stat.cached_num=MESA_htable_get_elem_num(keeper->cert_cache);
+    	// pthread_rwlock_unlock(&(keeper->rwlock));
+	}
+    *result = keeper->stat;
+    return;
+}
\ No newline at end of file
diff --git a/platform/src/ssl_stream.cpp b/platform/src/ssl_stream.cpp
index b2db7b6..1b151a4 100644
--- a/platform/src/ssl_stream.cpp
+++ b/platform/src/ssl_stream.cpp
@@ -352,7 +352,7 @@ ssl_stream_gc_cb(evutil_socket_t fd, short what, void * arg)
 		ssl_sess_cache_stat(mgr->up_sess_cache, &(mgr->stat_val[SSL_UP_CACHE_SZ]), &(mgr->stat_val[SSL_UP_CACHE_QUERY]), &(mgr->stat_val[SSL_UP_CACHE_HIT]));
 		ssl_sess_cache_stat(mgr->down_sess_cache, &(mgr->stat_val[SSL_DOWN_CACHE_SZ]), &(mgr->stat_val[SSL_DOWN_CACHE_QUERY]), &(mgr->stat_val[SSL_DOWN_CACHE_HIT]));
 	}
-	struct key_keeper_stat keeper_stat;
+	struct key_keeper_stat keeper_stat = { 0 };
 	key_keeper_statistic(mgr->key_keeper, &keeper_stat);
 	mgr->stat_val[KEY_KEEPER_ASK]=keeper_stat.ask_times;
 	mgr->stat_val[KEY_KEEPER_ISSUE]=keeper_stat.new_issue;