diff options
54 files changed, 34 insertions, 8443 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 16578e9..d1b1432 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -97,6 +97,5 @@ add_subdirectory(vendor) add_subdirectory(common) add_subdirectory(plugin) add_subdirectory(platform) -add_subdirectory(cache) add_subdirectory(script) add_subdirectory(bpf)
\ No newline at end of file diff --git a/cache/CMakeLists.txt b/cache/CMakeLists.txt deleted file mode 100644 index 9065655..0000000 --- a/cache/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -add_library(tango-cache-client src/object_store_client.cpp src/cache_evbase_client.cpp src/tango_cache_client.cpp src/tango_cache_redis.cpp src/tango_cache_pending.cpp src/tango_cache_tools.cpp src/tango_cache_transfer.cpp src/tango_cache_xml.cpp) -target_link_libraries(tango-cache-client http) -target_include_directories(tango-cache-client PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include) -target_link_libraries(tango-cache-client libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson) -target_link_libraries(tango-cache-client MESA_handle_logger MESA_htable MESA_prof_load wiredLB) diff --git a/cache/README.txt b/cache/README.txt deleted file mode 100644 index 535fe08..0000000 --- a/cache/README.txt +++ /dev/null @@ -1,12 +0,0 @@ -����Minio�ͼ�Ⱥ��Redis�����Ļ���ͻ��ˣ�����redis��ȡ�Ǹÿͻ���ʵ�֣�����Minio���¼�֪ͨredis�� -1��֧�����»��淽ʽ - ��1��Ԫ��Ϣ�Ͷ���ȫ���洢��Minio�� - ��2��Ԫ��Ϣ�洢��Redis������洢��Minio��˳��Ҳ�洢��Ԫ��Ϣ����HEAD������Redisȡ���� - ��3��Ԫ��Ϣ��С�ļ��洢��Redis�����ļ��洢��Minio����ֵ�������ļ��趨���� -2��������İ汾���£� - libevent��2.1�Ժ�汾�� - libcurl�� 7.43�Ժ�汾���Ƽ��汾7.59��һ��Ҫע��汾���⣡������������ - libxml2: 2.6�Ժ�汾 - hiredis-vip��0.3.0���ÿ��Ǿ����ĵİ汾���������صIJ���ֱ��ʹ�ã����������� - -3������ο�ͷ�ļ�tango_cache_client.h���ǽṹ����־API�ο�ͷ�ļ�cache_evbase_client.h��
\ No newline at end of file diff --git a/cache/include/cache_evbase_client.h b/cache/include/cache_evbase_client.h deleted file mode 100644 index b34e4eb..0000000 --- a/cache/include/cache_evbase_client.h +++ /dev/null @@ -1,67 +0,0 @@ -#ifndef __CACHE_ASYN_CLIENT_H__ -#define __CACHE_ASYN_CLIENT_H__ - -#include <event2/event.h> -#include <event.h> - -#include "tango_cache_client.h" - -/* API��ʹ��˵���ο�tango_cache_client.h */ - -struct cache_evbase_instance -{ - struct tango_cache_instance *instance; - evutil_socket_t notify_readfd; - evutil_socket_t notify_sendfd; - struct event_base* evbase; -}; - -struct cache_evbase_ctx -{ - size_t object_size; //tango_ctx�������Ա�����ع�����ֱ�ӻ�ȡ�Ļ����ڶ�дһ�������� - struct tango_cache_ctx *ctx; - struct cache_evbase_instance *instance_asyn; -}; - -/*����API�̰߳�ȫ������ͬһ��cache_evbase_ctx�����Կ��̷߳��ʡ�*/ - -enum CACHE_ERR_CODE cache_evbase_get_last_error(const struct cache_evbase_ctx *ctx_asyn); -enum CACHE_ERR_CODE cache_evbase_ctx_error(const struct cache_evbase_instance *instance); -void cache_evbase_get_statistics(const struct cache_evbase_instance *instance, struct cache_statistics *out); - -void cache_evbase_global_init(void); - -//ÿ��minio��Ⱥ��bucket����һ��parameter�����instance�ɹ���һ��parameter�� -struct tango_cache_parameter *cache_evbase_parameter_new(const char* profile_path, const char* section, void *runtimelog); - -/*����ʵ�����̰߳�ȫ���ڲ�������һ���߳�*/ -struct cache_evbase_instance *cache_evbase_instance_new(struct tango_cache_parameter *param, void *runtimelog); - -/******************************************* GET�ӿ� ****************************************/ -//�ɹ�����0��ʧ�ܷ���-1��future�ص���������������߳���ִ�У���ͬ -int cache_evbase_fetch_object(struct cache_evbase_instance *instance, struct future* f, struct tango_cache_meta_get *meta, enum OBJECT_LOCATION where_to_get); -int cache_evbase_head_object(struct cache_evbase_instance *instance, struct future* f, struct tango_cache_meta_get *meta); -struct tango_cache_result *cache_evbase_read_result(void *promise_result); - -/****************************************** DELETE�ӿ� **************************************/ -int cache_evbase_delete_object(struct cache_evbase_instance *instance, struct future* f, const char *objkey, const char *minio_addr=NULL, const char *bucket=NULL); - -/***************************************** һ�����ϴ��ӿ� **********************************/ -int cache_evbase_upload_once_data(struct cache_evbase_instance *instance, struct future* f, - enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, - struct tango_cache_meta_put *meta, - char *path/*OUT*/, size_t pathsize); -int cache_evbase_upload_once_evbuf(struct cache_evbase_instance *instance, struct future* f, - struct evbuffer *evbuf, - struct tango_cache_meta_put *meta, - char *path/*OUT*/, size_t pathsize); - -/****************************************** ��ʽ�ϴ��ӿ� ***********************************/ -struct cache_evbase_ctx *cache_evbase_update_start(struct cache_evbase_instance *instance, struct future* f, struct tango_cache_meta_put *meta); -int cache_evbase_update_frag_data(struct cache_evbase_ctx *ctx_asyn, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size); -int cache_evbase_update_frag_evbuf(struct cache_evbase_ctx *ctx_asyn, struct evbuffer *evbuf); -int cache_evbase_update_end(struct cache_evbase_ctx *ctx_asyn, char *path/*OUT*/, size_t pathsize); -void cache_evbase_update_cancel(struct cache_evbase_ctx *ctx_asyn); - -#endif - diff --git a/cache/include/object_store_client.h b/cache/include/object_store_client.h deleted file mode 100644 index 351bb83..0000000 --- a/cache/include/object_store_client.h +++ /dev/null @@ -1,52 +0,0 @@ -#ifndef __OBJECT_STORE_CLIENT_H__ -#define __OBJECT_STORE_CLIENT_H__ - -#include <event2/event.h> -#include <event.h> - -#include "cache_evbase_client.h" - -struct object_store_instance -{ - struct cache_evbase_instance **instances; - u_int32_t instance_num; -}; - -/*����API�̰߳�ȫ��API��ʹ��˵���ο�tango_cache_client.h*/ - -enum CACHE_ERR_CODE object_store_get_last_error(const struct cache_evbase_ctx *ctx_asyn); -void object_store_get_statistics(const struct object_store_instance *instance, struct cache_statistics *out); - -void object_store_global_init(void); - -/*����ʵ�����̰߳�ȫ���ڲ�������һ���߳�*/ -struct object_store_instance *object_store_instance_new(const char* profile_path, const char* section, int thread_num, void *runtimelog); - - -//GET�ӿڣ��ɹ�����0��ʧ�ܷ���-1��future�ص���������������߳���ִ�У���ͬ -int object_store_fetch_object(struct object_store_instance *instance, struct future* f, struct tango_cache_meta_get *meta, enum OBJECT_LOCATION where_to_get); -int object_store_head_object(struct object_store_instance *instance, struct future* f, struct tango_cache_meta_get *meta); -struct tango_cache_result *object_store_read_result(void *promise_result); - -//DELETE�ӿ� -int object_store_delete_object(struct object_store_instance *instance, struct future* f, const char *objkey, const char *minio_addr=NULL, const char *bucket=NULL); - -//һ�����ϴ��ӿ� -int object_store_upload_once_data(struct object_store_instance *instance, struct future* f, - enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, - struct tango_cache_meta_put *meta, - char *path/*OUT*/, size_t pathsize); -int object_store_upload_once_evbuf(struct object_store_instance *instance, struct future* f, - struct evbuffer *evbuf, - struct tango_cache_meta_put *meta, - char *path/*OUT*/, size_t pathsize); - -//��ʽ�ϴ��ӿ� -struct cache_evbase_ctx *object_store_update_start(struct object_store_instance *instance, struct future* f, struct tango_cache_meta_put *meta); -int object_store_update_frag_data(struct cache_evbase_ctx *ctx_asyn, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size); -int object_store_update_frag_evbuf(struct cache_evbase_ctx *ctx_asyn, struct evbuffer *evbuf); -int object_store_update_end(struct cache_evbase_ctx *ctx_asyn, char *path/*OUT*/, size_t pathsize); -void object_store_update_cancel(struct cache_evbase_ctx *ctx_asyn); - -#endif - diff --git a/cache/include/tango_cache_client.h b/cache/include/tango_cache_client.h deleted file mode 100644 index eac19bf..0000000 --- a/cache/include/tango_cache_client.h +++ /dev/null @@ -1,175 +0,0 @@ -#ifndef __TANGO_CACHE_CLIENT_H__ -#define __TANGO_CACHE_CLIENT_H__ - -#include <event2/event.h> -#include <event.h> - -#include <tfe_future.h> -#include "tango_cache_pending.h" - -#define USER_TAG_MAX_LEN 1518 - -enum CACHE_ERR_CODE -{ - CACHE_OK=0, - CACHE_CACHE_MISS = -101, //����δ���� - CACHE_TIMEOUT = -102, //���泬ʱ - CACHE_OUTOF_MEMORY= -103,//��ǰ�ڴ�ռ�ó������ƣ��鿴MAX_USED_MEMORY_SIZE_MB�Ƿ��С���ߵ�ǰ�ϴ����ʸ����ϵ����ߵ����� - CACHE_ERR_CURL = -104, - CACHE_ERR_WIREDLB = -105, - CACHE_ERR_SOCKPAIR= -106, - CACHE_ERR_INTERNAL= -107, - CACHE_ERR_REDIS_JSON = -108, - CACHE_ERR_REDIS_CONNECT= -109, - CACHE_ERR_REDIS_EXEC = -110, - CACHE_OUTOF_SESSION = -111, - CACHE_ERR_EVBUFFER = -112, - CACHE_UPDATE_CANCELED = -113, -}; - -struct cache_statistics -{ - long long get_recv_num; //����GET�Ĵ��� - long long get_succ_http; //GET minio�ɹ��Ĵ��� - long long get_succ_redis;//GET redis�ɹ��Ĵ��� - long long get_miss_num; //GETδ���еĴ��� - long long get_err_http; //GET minioʧ�ܵĴ��� - long long get_err_redis; //GET redisʧ�ܵĴ��� - long long put_recv_num; //����UPLOAD�Ĵ��� - long long put_succ_http; //UPLOAD minio�ɹ��Ĵ��� - long long put_succ_redis;//UPLOAD redis�ɹ��Ĵ��� - long long put_err_http; //UPLOAD minioʧ�ܵĴ��� - long long put_err_redis; //UPLOAD redisʧ�ܵĴ��� - long long del_recv_num; //����DELETE�Ĵ��� - long long del_succ_num; //DELETE�ɹ��Ĵ��� - long long del_error_num; //DELETEʧ�ܵĴ��� - long long totaldrop_num; //�ڴ����Լ�WiredLB����ʱDROP�Ĵ��� - long long memory_used; //��ǰUPLOAD BODY��ռ�ڴ��С - long long session_http; //��ǰ���ڽ���GET/PUT��HTTP�Ự�� - long long session_redis; //��ǰ���ڽ���GET/PUT��redis�Ự�� -}; - - -struct tango_cache_parameter; -struct tango_cache_instance; -struct tango_cache_ctx; - -enum CACHE_ERR_CODE tango_cache_get_last_error(const struct tango_cache_ctx *ctx); -enum CACHE_ERR_CODE tango_cache_ctx_error(const struct tango_cache_instance *instance); -void tango_cache_get_statistics(const struct tango_cache_instance *instance, struct cache_statistics *out); - -/*ÿ������ִ��һ�γ�ʼ��*/ -void tango_cache_global_init(void); - -//ÿ��minio��Ⱥ��bucket����һ��parameter�����instance�ɹ���һ��parameter -struct tango_cache_parameter *tango_cache_parameter_new(const char* profile_path, const char* section, void *runtimelog); -/*��������API�̲߳���ȫ*/ -//ÿ�������̴߳���һ��instance -struct tango_cache_instance *tango_cache_instance_new(struct tango_cache_parameter *param, struct event_base* evbase, void *runtimelog); - - -/****************************************** GET�ӿڵ�API ******************************************/ -enum CACHE_RESULT_TYPE -{ - RESULT_TYPE_HEADER=0, //���ֻ����һ�� - RESULT_TYPE_USERTAG, //���ֻ����һ�� - RESULT_TYPE_BODY, //���ܵ��ö�� - RESULT_TYPE_END, //ȫ��������ֻ����һ�Σ����������� - RESULT_TYPE_MISS, //����δ���У����������ͻ��⣬ֻ����һ��(��END֮��)������������ -}; - -enum OBJECT_LOCATION -{ - OBJECT_IN_UNKNOWN=0, - OBJECT_IN_HOS, - OBJECT_IN_REDIS -}; - -struct tango_cache_meta_get -{ - const char* url; //����:URL���ǽṹ����־:�ļ�·������CACHE_OBJECT_KEY_HASH_SWITCH=0ʱ���256�ֽڣ�=1ʱ������ - struct request_freshness get; -}; - -//promise_success�Ľ��result -struct tango_cache_result -{ - const char *data_frag; //���typeΪRESULT_TYPE_HEADER��ÿ��ͷ��������һ������(HTTP1.1��ʽ) - size_t size; - size_t tlength; //������ܳ��ȣ��ص�ʱ����Ч - enum CACHE_RESULT_TYPE type; - enum OBJECT_LOCATION location; -}; - -//�ɹ�ʱ�ص�promise_success -//ʧ��ʱ�ص�promise_failed(��һ��)��ʹ��get_last_error��ȡ�����룻 -//future������ΪNULL -int tango_cache_fetch_object(struct tango_cache_instance *instance, struct future* f, struct tango_cache_meta_get *meta, enum OBJECT_LOCATION where_to_get); -int tango_cache_head_object(struct tango_cache_instance *instance, struct future* f, struct tango_cache_meta_get *meta); -//��promise_success��result������ȡ��� -struct tango_cache_result *tango_cache_read_result(future_result_t *promise_result); - - -/****************************************** DELETE�ӿڵ�API ******************************************/ -int tango_cache_delete_object(struct tango_cache_instance *instance, struct future* f, const char *objkey, const char *minio_addr=NULL, const char *bucket=NULL); - - -/****************************************** UPLOAD�ӿڵ�API ******************************************/ -/* ע��: ��future��ΪNULL�������ϴ�����ʱ�����֪ͨ�ص������������ã�*/ - -enum PUT_MEMORY_COPY_WAY -{ - PUT_MEM_COPY=0, //��������ڴ� - PUT_MEM_FREE, //�������ڴ棬��������ɱ�����ģ���ͷŸ��ڴ� -}; -enum EVBUFFER_COPY_WAY -{ - EVBUFFER_MOVE=0,//evbuffer_add_buffer - EVBUFFER_COPY, //evbuffer_add_buffer_reference -}; - -enum CACHE_HTTP_HDR_TYPE -{ - HDR_CONTENT_TYPE=0, - HDR_CONTENT_ENCODING, - HDR_CONTENT_DISPOSITION, - HDR_CONTENT_MD5, - - HDR_CONTENT_NUM, -}; - -struct tango_cache_meta_put -{ - const char* url; - const char* std_hdr[HDR_CONTENT_NUM]; //����ͷ������"Content-Type: text/html"����Ҫ�������У�NULL��ʾû�и�ͷ���� - const char* usertag; //�������������ݣ�GETʱ��ԭ������ - size_t usertag_len; //���USER_TAG_MAX_LEN��0��ʾû�и�ͷ�� - size_t user_log_name; - struct response_freshness put; -}; - -/****************************************** ����һ��UPLOAD�ӿڵ�API ******************************************/ -//��path��Ϊ�գ����������Ĵ洢·�� -//����ֵ: 0-�ɹ���<0ʧ�ܣ���ͬ -int tango_cache_upload_once_data(struct tango_cache_instance *instance, struct future* f, - enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, - struct tango_cache_meta_put *meta, - char *path/*OUT*/, size_t pathsize); -int tango_cache_upload_once_evbuf(struct tango_cache_instance *instance, struct future* f, - enum EVBUFFER_COPY_WAY way, struct evbuffer *evbuf, - struct tango_cache_meta_put *meta, - char *path/*OUT*/, size_t pathsize); - -/****************************************** ��ʽUPLOAD�ӿڵ�API ******************************************/ -//����ֵ: ��ΪNULL���ʾ����ʧ�ܣ�����tango_cache_ctx_error�鿴�������Ƿ���CACHE_OUTOF_MEMORY(�����������)�� -struct tango_cache_ctx *tango_cache_update_start(struct tango_cache_instance *instance, struct future* f, struct tango_cache_meta_put *meta); -//����ֵ: 0-�ɹ���<0ʧ�ܣ�����tango_cache_get_last_error�鿴�����룻 -int tango_cache_update_frag_data(struct tango_cache_ctx *ctx, const char *data, size_t size); -int tango_cache_update_frag_evbuf(struct tango_cache_ctx *ctx, enum EVBUFFER_COPY_WAY way, struct evbuffer *evbuf); -//ע��: ����ʧ��ʱ���ٵ���promise�ص�������path��ΪNULLʱ���ش洢·�� -int tango_cache_update_end(struct tango_cache_ctx *ctx, char *path/*OUT*/, size_t pathsize); -//����cancel���������ص�����������ʧ�� -void tango_cache_update_cancel(struct tango_cache_ctx *ctx); - -#endif - diff --git a/cache/include/tango_cache_pending.h b/cache/include/tango_cache_pending.h deleted file mode 100644 index 0c72635..0000000 --- a/cache/include/tango_cache_pending.h +++ /dev/null @@ -1,64 +0,0 @@ -#pragma once - -#include<time.h> -#include<tfe_http.h> - -enum cache_pending_action { - UNDEFINED = 0, - ALLOWED, - FORBIDDEN, - REVALIDATE -}; - - -struct tfe_http_field { - - enum tfe_http_std_field http_field; - const char* value; -}; - - -struct request_freshness { - time_t min_fresh; - time_t max_age; -}; - - -struct response_freshness{ - time_t date; - time_t last_modified; - time_t timeout; -}; - - -time_t read_GMT_time(const char* gmt_string); -/* -函数功能： -根据请求头字段判断是否允许将缓存作为该请求的响应，并且将请求字段对缓存新鲜度的约束范围作为传出参数返回给调用者 -参数: -request:HTTP请求字段信息，包括Pragma,Cache-Control,If-Match,If-None-Match,If-Modified-Since,If-Unmodified-Since字段 -n_fields:request包含的HTTP请求字段数目 -restrict:如果该函数返回值为ALLOWED，则返回请求Cache-Control字段包括的min-fresh或者max-age值；否则返回值为0 -返回值： -UNDEFINED = 0,//请求字段中未定义缓存的行为 -ALLOWED ,//允许使用缓存作为该请求的响应 -FORBIDDEN,//禁止使用缓存作为该请求的响应，需要向源服务器请求 -REVALIDATE,//禁止使用未验证有效性的缓存作为该请求的响应 -*/ -enum cache_pending_action tfe_cache_get_pending(const struct tfe_http_half *request, struct request_freshness* restrict); - - - -/* -函数功能： -根据响应字段判断该响应是否允许被缓存，并且将响应的新鲜度、date或者last-modified值作为传出参数返回给调用者 -参数: -response:HTTP响应字段信息，包括Pragma,Cache-Control,Expires、Date、Last-Modified字段 -n_fields:response包含的HTTP响应字段数目 -freshness:如果该函数返回值为ALLOWED，则返回响应Cache-Control字段的s-maxage/max-age值以及Date字段、Last-Modified字段相对时间 -返回值： -UNDEFINED = 0,//响应字段中未定义缓存的行为 -ALLOWED ,//允许缓存该响应 -FORBIDDEN,//禁止缓存该响应 -*/ -enum cache_pending_action tfe_cache_put_pending(const struct tfe_http_half *response, struct response_freshness* freshness); diff --git a/cache/src/cache_evbase_client.cpp b/cache/src/cache_evbase_client.cpp deleted file mode 100644 index e7835ef..0000000 --- a/cache/src/cache_evbase_client.cpp +++ /dev/null @@ -1,640 +0,0 @@ -#include <sys/ioctl.h> -#include <sys/socket.h> -#include <sys/types.h> -#include <netinet/in.h> -#include <netinet/tcp.h> -#include <arpa/inet.h> -#include <net/if.h> -#include <unistd.h> -#include <stdio.h> -#include <stdlib.h> -#include <assert.h> -#include <errno.h> -#include <sys/prctl.h> -#include <string.h> -#include <pthread.h> - -#include "cache_evbase_client.h" -#include "tango_cache_transfer.h" -#include "tango_cache_tools.h" - -enum CACHE_ASYN_CMD -{ - CACHE_ASYN_FETCH=0, - CACHE_ASYN_UPLOAD_ONCE_DATA, - CACHE_ASYN_UPLOAD_ONCE_EVBUF, - CACHE_ASYN_UPLOAD_START, - CACHE_ASYN_UPLOAD_FRAG_DATA, - CACHE_ASYN_UPLOAD_FRAG_EVBUF, - CACHE_ASYN_UPLOAD_END, - CACHE_ASYN_UPLOAD_CANCEL, - CACHE_ASYN_DELETE, - CACHE_ASYN_HEAD, -}; - -struct databuffer -{ - char *data; - size_t size; - struct evbuffer *evbuf; - enum CACHE_ASYN_CMD cmd_type; - enum OBJECT_LOCATION where_to_get; - struct cache_evbase_ctx *ctx_asyn; -}; - -enum CACHE_ERR_CODE cache_evbase_get_last_error(const struct cache_evbase_ctx *ctx_asyn) -{ - return tango_cache_get_last_error(ctx_asyn->ctx); -} -enum CACHE_ERR_CODE cache_evbase_ctx_error(const struct cache_evbase_instance *instance) -{ - return tango_cache_ctx_error(instance->instance); -} - -void cache_evbase_get_statistics(const struct cache_evbase_instance *instance, struct cache_statistics *out) -{ - tango_cache_get_statistics(instance->instance, out); -} - -struct tango_cache_result *cache_evbase_read_result(void *promise_result) -{ - return tango_cache_read_result(promise_result); -} - -static int create_notification_pipe(evutil_socket_t sv[2]) -{ - if(evutil_socketpair(AF_LOCAL, SOCK_STREAM, 0, sv) == -1) - { - return -1; - } - if(evutil_make_socket_nonblocking(sv[0])<0 || evutil_make_socket_nonblocking(sv[1])<0) - { - return -1; - } - if(evutil_make_socket_closeonexec(sv[0])<0 || evutil_make_socket_closeonexec(sv[1])<0) - { - return -1; - } - return 0; -} - -static int32_t mesa_tcp_sock_write (int32_t write_fd, void *buf, int32_t bufsize) -{ - int32_t res; - - do{ - res = send(write_fd, buf, bufsize, MSG_NOSIGNAL); - }while (res==-1 &&(errno == EINTR)); - - return res; -} - -static int32_t iothread_notify_event(int32_t socket_fd, void *content, int32_t len, int32_t s_time_out) -{ - fd_set w_set, e_set; - struct timeval tv; - int32_t res=0, sndlen=0, sendsize=0; - - while(len > sndlen) - { - FD_ZERO (&w_set); - FD_ZERO (&e_set); - FD_SET (socket_fd, &w_set); - FD_SET (socket_fd, &e_set); - if(s_time_out == 0) - { - res = select (socket_fd + 1, NULL, &w_set, &e_set, NULL); - } - else - { - tv.tv_sec = s_time_out; - tv.tv_usec = 0; - res = select (socket_fd + 1, NULL, &w_set, &e_set, &tv); - } - if(res <= 0) - { - printf("log_error: select io res=%d, error: %s\n", res, strerror(errno)); - return -1; - } - - if(FD_ISSET(socket_fd, &e_set)) - { - printf("log_error: select io is in efds, error: %s\n", strerror(errno)); - return -2; - } - - if(FD_ISSET(socket_fd, &w_set)) - { - sendsize = mesa_tcp_sock_write(socket_fd, (char*)content + sndlen, len - sndlen); - if (sendsize < 0) - { - if(errno == EAGAIN) - { - continue; - } - return -1; - } - sndlen += sendsize; - } - } - - return sndlen; -} - -static void cache_asyn_ctx_destroy(struct cache_evbase_ctx *ctx_asyn) -{ - free(ctx_asyn); -} - -static void cache_asyn_ioevent_dispatch(struct databuffer *buffer) -{ - struct cache_evbase_ctx *ctx_asyn=buffer->ctx_asyn; - struct promise *p; - int ret=0; - - switch(buffer->cmd_type) - { - case CACHE_ASYN_FETCH: - p = ctx_asyn->ctx->promise; - if(do_tango_cache_fetch_object(ctx_asyn->ctx, buffer->where_to_get) < 0) - { - promise_failed(p, FUTURE_ERROR_CANCEL, "CACHE_ASYN_FETCH failed"); - } - cache_asyn_ctx_destroy(ctx_asyn); - break; - case CACHE_ASYN_HEAD: - p = ctx_asyn->ctx->promise; - ret = do_tango_cache_head_object(ctx_asyn->ctx, buffer->where_to_get); - if(ret<0) - { - promise_failed(p, FUTURE_ERROR_CANCEL, "CACHE_ASYN_HEAD failed"); - } - cache_asyn_ctx_destroy(ctx_asyn); - break; - - case CACHE_ASYN_DELETE: - cache_delete_minio_object(ctx_asyn->ctx, true); - cache_asyn_ctx_destroy(ctx_asyn); - break; - - case CACHE_ASYN_UPLOAD_ONCE_DATA: - do_tango_cache_upload_once_data(ctx_asyn->ctx, PUT_MEM_FREE, buffer->data, buffer->size, true); - cache_asyn_ctx_destroy(ctx_asyn); - break; - case CACHE_ASYN_UPLOAD_ONCE_EVBUF: - do_tango_cache_upload_once_evbuf(ctx_asyn->ctx, EVBUFFER_MOVE, buffer->evbuf, true); - evbuffer_free(buffer->evbuf); - cache_asyn_ctx_destroy(ctx_asyn); - break; - - case CACHE_ASYN_UPLOAD_START: - ctx_asyn->ctx->instance->statistic.put_recv_num += 1; - ctx_asyn->ctx->instance->error_code = CACHE_OK; - break; - - case CACHE_ASYN_UPLOAD_FRAG_DATA: - tango_cache_update_frag_data(ctx_asyn->ctx, buffer->data, buffer->size); - free(buffer->data); - break; - - case CACHE_ASYN_UPLOAD_FRAG_EVBUF: - tango_cache_update_frag_evbuf(ctx_asyn->ctx, EVBUFFER_MOVE, buffer->evbuf); - evbuffer_free(buffer->evbuf); - break; - - case CACHE_ASYN_UPLOAD_END: - do_tango_cache_update_end(ctx_asyn->ctx, true); - cache_asyn_ctx_destroy(ctx_asyn); - break; - case CACHE_ASYN_UPLOAD_CANCEL: - tango_cache_update_cancel(ctx_asyn->ctx); - cache_asyn_ctx_destroy(ctx_asyn); - break; - default: assert(0);break; - } -} - -static void sockpair_notification_handler(evutil_socket_t fd, short events, void *arg) -{ - ssize_t readlen, needlen; - struct cache_evbase_instance *instance_asyn = (struct cache_evbase_instance *)arg; - struct databuffer *buffer; - - while(1) - { - needlen=sizeof(struct cache_evbase_ctx *); - readlen = recv(fd, &buffer, needlen, 0); - if(readlen == needlen) - { - cache_asyn_ioevent_dispatch(buffer); - free(buffer); - } - else - { - if(errno!=EWOULDBLOCK && errno!=EAGAIN) - { - MESA_HANDLE_RUNTIME_LOGV2(instance_asyn->instance->runtime_log, RLOG_LV_FATAL, "read pipe error: %s.", strerror(errno)); - assert(0); - return; - } - break; - } - } -} - -static void* thread_listen_sockpair(void *arg) -{ - struct cache_evbase_instance *instance_asyn = (struct cache_evbase_instance *)arg; - struct event listen_event; - - prctl(PR_SET_NAME, "tango_cache"); - - event_assign(&listen_event, instance_asyn->evbase, instance_asyn->notify_readfd, EV_READ|EV_PERSIST, sockpair_notification_handler, instance_asyn); - event_add(&listen_event, NULL); - event_base_dispatch(instance_asyn->evbase); - - printf("Libevent dispath error, should not run here.\n"); - MESA_HANDLE_RUNTIME_LOGV2(instance_asyn->instance->runtime_log, RLOG_LV_FATAL, "Libevent dispath error, should not run here."); - assert(0); - return NULL; -} - -int cache_evbase_update_end(struct cache_evbase_ctx *ctx_asyn, char *path/*OUT*/, size_t pathsize) -{ - struct databuffer *buffer; - - if(ctx_asyn->ctx->fail_state) - { - tango_cache_ctx_destroy(ctx_asyn->ctx, false); - cache_asyn_ctx_destroy(ctx_asyn); - return -1; - } - buffer = (struct databuffer *)malloc(sizeof(struct databuffer)); - buffer->ctx_asyn = ctx_asyn; - buffer->cmd_type = CACHE_ASYN_UPLOAD_END; - - //ENDʱ����δ��ʼ�ֶ��ϴ��������ϴ�֮ǰ����locateһ��λ�� - ctx_asyn->ctx->locate = tango_cache_object_locate(ctx_asyn->ctx->instance, ctx_asyn->object_size); - tango_cache_get_object_path(ctx_asyn->ctx, path, pathsize); - if(ctx_asyn->ctx->instance->param->object_store_way != CACHE_ALL_HOS) - { - cJSON_AddNumberToObject(ctx_asyn->ctx->put.object_meta, "Content-Length", ctx_asyn->object_size); - } - - if(iothread_notify_event(ctx_asyn->instance_asyn->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *)) - { - tango_cache_set_fail_state(ctx_asyn->ctx, CACHE_ERR_SOCKPAIR); - tango_cache_ctx_destroy(ctx_asyn->ctx, false); - cache_asyn_ctx_destroy(ctx_asyn); - free(buffer); - return -2; - } - return 0; -} - -void cache_evbase_update_cancel(struct cache_evbase_ctx *ctx_asyn) -{ - struct databuffer *buffer; - - buffer = (struct databuffer *)malloc(sizeof(struct databuffer)); - buffer->ctx_asyn = ctx_asyn; - buffer->cmd_type = CACHE_ASYN_UPLOAD_CANCEL; - - if(iothread_notify_event(ctx_asyn->instance_asyn->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *)) - { - if(!ctx_asyn->ctx->fail_state) - { - tango_cache_set_fail_state(ctx_asyn->ctx, CACHE_ERR_SOCKPAIR); - } - tango_cache_ctx_destroy(ctx_asyn->ctx, false); - cache_asyn_ctx_destroy(ctx_asyn); - free(buffer); - } -} - -int cache_evbase_update_frag_data(struct cache_evbase_ctx *ctx_asyn, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size) -{ - struct databuffer *buffer; - - if(ctx_asyn->ctx->fail_state) - { - if(way == PUT_MEM_FREE) free((void *)data); - return 0;//��ʱ�Ⱥ��Է���ֵ���Իص���ʽ֪ͨ����������û���֪��ʱEND�����⡣ - } - ctx_asyn->object_size += size; - - buffer = (struct databuffer *)malloc(sizeof(struct databuffer)); - if(way == PUT_MEM_COPY) - { - buffer->data = (char *)malloc(size); - memcpy(buffer->data, data, size); - } - else - { - buffer->data = (char*)data; - } - buffer->size = size; - buffer->ctx_asyn = ctx_asyn; - buffer->cmd_type = CACHE_ASYN_UPLOAD_FRAG_DATA; - - if(iothread_notify_event(ctx_asyn->instance_asyn->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *)) - { - tango_cache_set_fail_state(ctx_asyn->ctx, CACHE_ERR_SOCKPAIR); - free(buffer->data); - free(buffer); - return -2; - } - return 0; -} - -int cache_evbase_update_frag_evbuf(struct cache_evbase_ctx *ctx_asyn, struct evbuffer *evbuf) -{ - struct databuffer *buffer; - - if(ctx_asyn->ctx->fail_state) - { - return 0; - } - ctx_asyn->object_size += evbuffer_get_length(evbuf); - buffer = (struct databuffer *)malloc(sizeof(struct databuffer)); - buffer->ctx_asyn = ctx_asyn; - buffer->cmd_type = CACHE_ASYN_UPLOAD_FRAG_EVBUF; - buffer->evbuf = evbuffer_new(); - evbuffer_add_buffer(buffer->evbuf, evbuf); - - if(iothread_notify_event(ctx_asyn->instance_asyn->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *)) - { - tango_cache_set_fail_state(ctx_asyn->ctx, CACHE_ERR_SOCKPAIR); - evbuffer_free(buffer->evbuf); - free(buffer); - return -2; - } - return 0; -} - -struct cache_evbase_ctx *cache_evbase_update_start(struct cache_evbase_instance *instance, struct future* f, struct tango_cache_meta_put *meta) -{ - struct cache_evbase_ctx *ctx_asyn; - struct tango_cache_ctx *ctx; - struct databuffer *buffer; - enum OBJECT_LOCATION maybe_loc=OBJECT_IN_UNKNOWN; - - if(instance->instance->param->object_store_way != CACHE_SMALL_REDIS) - { - maybe_loc = OBJECT_IN_HOS; - } - ctx = tango_cache_update_prepare(instance->instance, f, meta, maybe_loc); - if(ctx == NULL) - { - return NULL; - } - ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx)); - ctx_asyn->instance_asyn = instance; - ctx_asyn->ctx = ctx; - - buffer = (struct databuffer *)malloc(sizeof(struct databuffer)); - buffer->ctx_asyn = ctx_asyn; - buffer->cmd_type = CACHE_ASYN_UPLOAD_START; - - //�¼�֪ͨ��Ϊ������ͳ����Ϣ - if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *)) - { - instance->instance->error_code = CACHE_ERR_SOCKPAIR; - tango_cache_set_fail_state(ctx, CACHE_ERR_SOCKPAIR); - tango_cache_ctx_destroy(ctx, false); - cache_asyn_ctx_destroy(ctx_asyn); - free(buffer); - return NULL; - } - return ctx_asyn; -} - -int cache_evbase_upload_once_data(struct cache_evbase_instance *instance, struct future* f, - enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, struct tango_cache_meta_put *meta, char *path, size_t pathsize) -{ - struct cache_evbase_ctx *ctx_asyn; - struct tango_cache_ctx *ctx; - struct databuffer *buffer; - - ctx = tango_cache_update_once_prepare(instance->instance, f, meta, size, path, pathsize); - if(ctx == NULL) - { - if(way == PUT_MEM_FREE) free((void *)data); - return -1; - } - ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx)); - ctx_asyn->instance_asyn = instance; - ctx_asyn->ctx = ctx; - - buffer = (struct databuffer *)malloc(sizeof(struct databuffer)); - if(way == PUT_MEM_COPY) - { - buffer->data = (char *)malloc(size); - memcpy(buffer->data, data, size); - } - else - { - buffer->data = (char*)data; - } - buffer->size = size; - buffer->ctx_asyn = ctx_asyn; - buffer->cmd_type = CACHE_ASYN_UPLOAD_ONCE_DATA; - - if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *)) - { - free(buffer->data); - free(buffer); - instance->instance->error_code = CACHE_ERR_SOCKPAIR; - tango_cache_set_fail_state(ctx, CACHE_ERR_SOCKPAIR); - tango_cache_ctx_destroy(ctx, false); - cache_asyn_ctx_destroy(ctx_asyn); - return -2; - } - return 0; -} - -int cache_evbase_upload_once_evbuf(struct cache_evbase_instance *instance, struct future* f, - struct evbuffer *evbuf, struct tango_cache_meta_put *meta, char *path, size_t pathsize) -{ - struct cache_evbase_ctx *ctx_asyn; - struct tango_cache_ctx *ctx; - struct databuffer *buffer; - - ctx = tango_cache_update_once_prepare(instance->instance, f, meta, evbuffer_get_length(evbuf), path, pathsize); - if(ctx == NULL) - { - return -1; - } - ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx)); - ctx_asyn->instance_asyn = instance; - ctx_asyn->ctx = ctx; - - buffer = (struct databuffer *)malloc(sizeof(struct databuffer)); - buffer->ctx_asyn = ctx_asyn; - buffer->cmd_type = CACHE_ASYN_UPLOAD_ONCE_EVBUF; - buffer->evbuf = evbuffer_new(); - evbuffer_add_buffer(buffer->evbuf, evbuf); - - if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *)) - { - evbuffer_free(buffer->evbuf); - free(buffer); - instance->instance->error_code = CACHE_ERR_SOCKPAIR; - tango_cache_set_fail_state(ctx, CACHE_ERR_SOCKPAIR); - tango_cache_ctx_destroy(ctx, false); - cache_asyn_ctx_destroy(ctx_asyn); - return -2; - } - return 0; -} - -int cache_evbase_fetch_object(struct cache_evbase_instance *instance, struct future* f, struct tango_cache_meta_get *meta, enum OBJECT_LOCATION where_to_get) -{ - struct cache_evbase_ctx *ctx_asyn; - struct databuffer *buffer; - - if(instance->instance->param->object_store_way != CACHE_SMALL_REDIS) - { - where_to_get = OBJECT_IN_HOS; - } - ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx)); - ctx_asyn->instance_asyn = instance; - ctx_asyn->ctx = tango_cache_fetch_prepare(instance->instance, CACHE_REQUEST_GET, f, meta, where_to_get); - if(ctx_asyn->ctx == NULL) - { - free(ctx_asyn); - return -1; - } - - buffer = (struct databuffer *)malloc(sizeof(struct databuffer)); - buffer->ctx_asyn = ctx_asyn; - buffer->cmd_type = CACHE_ASYN_FETCH; - buffer->where_to_get = where_to_get; - - if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *)) - { - instance->instance->error_code = CACHE_ERR_SOCKPAIR; - tango_cache_set_fail_state(ctx_asyn->ctx, CACHE_ERR_SOCKPAIR); - tango_cache_ctx_destroy(ctx_asyn->ctx, false); - cache_asyn_ctx_destroy(ctx_asyn); - free(buffer); - return -2; - } - return 0; -} - -int cache_evbase_head_object(struct cache_evbase_instance *instance, struct future* f, struct tango_cache_meta_get *meta) -{ - struct cache_evbase_ctx *ctx_asyn; - struct databuffer *buffer; - enum OBJECT_LOCATION location = OBJECT_IN_HOS; - - if(instance->instance->param->object_store_way != CACHE_ALL_HOS) - { - location = OBJECT_IN_REDIS; - } - ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx)); - ctx_asyn->instance_asyn = instance; - ctx_asyn->ctx = tango_cache_fetch_prepare(instance->instance, CACHE_REQUEST_HEAD, f, meta, location); - if(ctx_asyn->ctx == NULL) - { - free(ctx_asyn); - return -1; - } - - buffer = (struct databuffer *)malloc(sizeof(struct databuffer)); - buffer->ctx_asyn = ctx_asyn; - buffer->cmd_type = CACHE_ASYN_HEAD; - buffer->where_to_get = location; - - if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *)) - { - instance->instance->error_code = CACHE_ERR_SOCKPAIR; - tango_cache_set_fail_state(ctx_asyn->ctx, CACHE_ERR_SOCKPAIR); - tango_cache_ctx_destroy(ctx_asyn->ctx, false); - cache_asyn_ctx_destroy(ctx_asyn); - free(buffer); - return -2; - } - return 0; -} - -int cache_evbase_delete_object(struct cache_evbase_instance *instance, struct future* f, const char *objkey, const char *minio_addr, const char *bucket) -{ - struct cache_evbase_ctx *ctx_asyn; - struct databuffer *buffer; - - ctx_asyn = (struct cache_evbase_ctx *)calloc(1, sizeof(struct cache_evbase_ctx)); - ctx_asyn->instance_asyn = instance; - ctx_asyn->ctx = tango_cache_delete_prepare(instance->instance, f, objkey, minio_addr, bucket); - if(ctx_asyn->ctx == NULL) - { - free(ctx_asyn); - return -1; - } - - buffer = (struct databuffer *)malloc(sizeof(struct databuffer)); - buffer->ctx_asyn = ctx_asyn; - buffer->cmd_type = CACHE_ASYN_DELETE; - - //�ο�Unix�����432ҳ���ڶ��߳�д�İ�ȫ������ - if(iothread_notify_event(instance->notify_sendfd, &buffer, sizeof(void *), 2) != sizeof(void *)) - { - instance->instance->error_code = CACHE_ERR_SOCKPAIR; - tango_cache_set_fail_state(ctx_asyn->ctx, CACHE_ERR_SOCKPAIR); - tango_cache_ctx_destroy(ctx_asyn->ctx, false); - cache_asyn_ctx_destroy(ctx_asyn); - free(buffer); - return -2; - } - return 0; -} - -struct tango_cache_parameter *cache_evbase_parameter_new(const char* profile_path, const char* section, void *runtimelog) -{ - return tango_cache_parameter_new(profile_path, section, runtimelog); -} - -struct cache_evbase_instance *cache_evbase_instance_new(struct tango_cache_parameter *param, void *runtimelog) -{ - evutil_socket_t notification_fd[2]; - struct cache_evbase_instance *instance_asyn; - struct event_base *evbase; - pthread_t thread_tid; - pthread_attr_t attr; - - if(create_notification_pipe(notification_fd)) - { - return NULL; - } - if((evbase = event_base_new()) == NULL) - { - return NULL; - } - - instance_asyn = (struct cache_evbase_instance *)calloc(1, sizeof(struct cache_evbase_instance)); - instance_asyn->evbase = evbase; - instance_asyn->notify_readfd = notification_fd[0]; - instance_asyn->notify_sendfd = notification_fd[1]; - instance_asyn->instance = tango_cache_instance_new(param, evbase, runtimelog); - if(instance_asyn->instance == NULL) - { - free(instance_asyn); - return NULL; - } - - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - if(pthread_create(&thread_tid, &attr, thread_listen_sockpair, instance_asyn)) - { - free(instance_asyn); - return NULL; - } - return instance_asyn; -} - -void cache_evbase_global_init(void) -{ - tango_cache_global_init(); -} - diff --git a/cache/src/object_store_client.cpp b/cache/src/object_store_client.cpp deleted file mode 100644 index 58eb3fb..0000000 --- a/cache/src/object_store_client.cpp +++ /dev/null @@ -1,145 +0,0 @@ -#include <sys/ioctl.h> -#include <sys/socket.h> -#include <sys/types.h> -#include <netinet/in.h> -#include <netinet/tcp.h> -#include <arpa/inet.h> -#include <net/if.h> -#include <unistd.h> -#include <stdio.h> -#include <stdlib.h> -#include <assert.h> -#include <errno.h> -#include <sys/prctl.h> -#include <string.h> -#include <pthread.h> - -#include "object_store_client.h" -#include "tango_cache_tools.h" - -enum CACHE_ERR_CODE object_store_get_last_error(const struct cache_evbase_ctx *ctx) -{ - return cache_evbase_get_last_error(ctx); -} - -void object_store_get_statistics(const struct object_store_instance *instance, struct cache_statistics *out) -{ - struct cache_statistics out_cache; - - memset(out, 0, sizeof(struct cache_statistics)); - for(u_int32_t i=0; i<instance->instance_num; i++) - { - cache_evbase_get_statistics(instance->instances[i], &out_cache); - - out->del_error_num += out_cache.del_error_num; - out->del_recv_num += out_cache.del_recv_num; - out->del_succ_num += out_cache.del_succ_num; - out->get_err_http += out_cache.get_err_http; - out->get_err_redis += out_cache.get_err_redis; - out->get_miss_num += out_cache.get_miss_num; - out->get_recv_num += out_cache.get_recv_num; - out->get_succ_http += out_cache.get_succ_http; - out->get_succ_redis+= out_cache.get_succ_redis; - out->put_err_http += out_cache.put_err_http; - out->put_err_redis += out_cache.put_err_redis; - out->put_recv_num += out_cache.put_recv_num; - out->put_succ_http += out_cache.put_succ_http; - out->put_succ_redis+= out_cache.put_succ_redis; - out->session_http += out_cache.session_http; - out->session_redis += out_cache.session_redis; - out->memory_used += out_cache.memory_used; - out->totaldrop_num += out_cache.totaldrop_num; - } -} - -struct tango_cache_result *object_store_read_result(void *promise_result) -{ - return cache_evbase_read_result(promise_result); -} - -int object_store_update_end(struct cache_evbase_ctx *ctx, char *path/*OUT*/, size_t pathsize) -{ - return cache_evbase_update_end(ctx, path, pathsize); -} - -void object_store_update_cancel(struct cache_evbase_ctx *ctx) -{ - cache_evbase_update_cancel(ctx); -} - -int object_store_update_frag_data(struct cache_evbase_ctx *ctx, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size) -{ - return cache_evbase_update_frag_data(ctx, way, data, size); -} - -int object_store_update_frag_evbuf(struct cache_evbase_ctx *ctx, struct evbuffer *evbuf) -{ - return cache_evbase_update_frag_evbuf(ctx, evbuf); -} - -struct cache_evbase_ctx *object_store_update_start(struct object_store_instance *instance, struct future* f, struct tango_cache_meta_put *meta) -{ - return cache_evbase_update_start(instance->instances[rand()%instance->instance_num], f, meta); -} - -int object_store_upload_once_data(struct object_store_instance *instance, struct future* f, - enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, struct tango_cache_meta_put *meta, char *path, size_t pathsize) -{ - return cache_evbase_upload_once_data(instance->instances[rand()%instance->instance_num], f, way, data, size, meta, path, pathsize); -} - -int object_store_upload_once_evbuf(struct object_store_instance *instance, struct future* f, - struct evbuffer *evbuf, struct tango_cache_meta_put *meta, char *path, size_t pathsize) -{ - return cache_evbase_upload_once_evbuf(instance->instances[rand()%instance->instance_num], f, evbuf, meta, path, pathsize); -} - -int object_store_fetch_object(struct object_store_instance *instance, struct future* f, struct tango_cache_meta_get *meta, enum OBJECT_LOCATION where_to_get) -{ - return cache_evbase_fetch_object(instance->instances[rand()%instance->instance_num], f, meta, where_to_get); -} - -int object_store_head_object(struct object_store_instance *instance, struct future* f, struct tango_cache_meta_get *meta) -{ - return cache_evbase_head_object(instance->instances[rand()%instance->instance_num], f, meta); -} - -int object_store_delete_object(struct object_store_instance *instance, struct future* f, const char *objkey, const char *minio_addr, const char *bucket) -{ - return cache_evbase_delete_object(instance->instances[rand()%instance->instance_num], f, objkey, minio_addr, bucket); -} - -struct object_store_instance *object_store_instance_new(const char* profile_path, const char* section, int thread_num, void *runtimelog) -{ - struct object_store_instance *object_instance; - struct tango_cache_parameter *parameter; - - parameter = tango_cache_parameter_new(profile_path, section, runtimelog); - if(parameter == NULL) - { - return NULL; - } - - object_instance = (struct object_store_instance *)calloc(1, sizeof(struct object_store_instance)); - object_instance->instance_num = thread_num; - object_instance->instances = (struct cache_evbase_instance **)calloc(1, sizeof(struct cache_evbase_instance *)*object_instance->instance_num); - - for(u_int32_t i=0; i<object_instance->instance_num; i++) - { - object_instance->instances[i] = cache_evbase_instance_new(parameter, runtimelog); - if(object_instance->instances[i] == NULL) - { - free(parameter); - free(object_instance); - return NULL; - } - } - srandom(time(NULL)); - return object_instance; -} - -void object_store_global_init(void) -{ - cache_evbase_global_init(); -} - diff --git a/cache/src/tango_cache_client.cpp b/cache/src/tango_cache_client.cpp deleted file mode 100644 index 07674ad..0000000 --- a/cache/src/tango_cache_client.cpp +++ /dev/null @@ -1,1251 +0,0 @@ -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> -#include <stdio.h> -#include <stdlib.h> -#include <assert.h> -#include <errno.h> -#include <sys/time.h> -#include <time.h> -#include <string.h> -#include <openssl/sha.h> -#include <openssl/md5.h> - -#include <MESA/MESA_prof_load.h> - -#include "tango_cache_client_in.h" -#include "tango_cache_transfer.h" -#include "tango_cache_tools.h" -#include "tango_cache_xml.h" -#include "tango_cache_redis.h" - -int TANGO_CACHE_VERSION_20181009=0; - -static int caculate_base64_md5(const char *data, unsigned long len, unsigned char *result, unsigned int size) -{ - MD5_CTX c; - unsigned char md5[17]={0}; - - if(size < 33) - return -1; - - MD5_Init(&c); - MD5_Update(&c, data, len); - MD5_Final(md5, &c); - - Base64_EncodeBlock(md5, 16, result); - return 0; -} - -void caculate_sha256(const char *data, unsigned long len, char *result, u_int32_t size) -{ - SHA256_CTX c; - unsigned char sha256[128]; - u_int32_t length; - - SHA256_Init(&c); - SHA256_Update(&c, data, len); - SHA256_Final(sha256, &c); - - length = (size > 64)?32:(size-1)/2; //Ԥ��һ���ռ� - for(u_int32_t i=0; i<length; i++) - { - sprintf(result + i*2, "%02x", sha256[i]); - } - result[length*2] = '\0'; -} - -static int wired_load_balancer_lookup(WLB_handle_t wiredlb, const char *key, int keylen, char *host, size_t hostsize) -{ - struct WLB_consumer_t chosen; - - if(wiredLB_lookup(wiredlb, key, keylen, &chosen)) - { - return -1; - } - snprintf(host, hostsize, "%s:%hu", chosen.ip_addr, chosen.data_port); - return 0; -} - -enum CACHE_ERR_CODE tango_cache_get_last_error(const struct tango_cache_ctx *ctx) -{ - return ctx->error_code; -} -enum CACHE_ERR_CODE tango_cache_ctx_error(const struct tango_cache_instance *instance) -{ - return instance->error_code; -} - -void tango_cache_set_fail_state(struct tango_cache_ctx *ctx, enum CACHE_ERR_CODE error_code) -{ - ctx->fail_state = true; - ctx->error_code = error_code; -} - -const char *tango_cache_get_errstring(const struct tango_cache_ctx *ctx) -{ - switch(ctx->error_code) - { - case CACHE_CACHE_MISS: return "cache not hit"; - case CACHE_TIMEOUT: return "cache not fresh"; - case CACHE_OUTOF_MEMORY: return "outof memory"; - case CACHE_ERR_WIREDLB: return "wiredlb error"; - case CACHE_ERR_SOCKPAIR: return "socketpair error"; - case CACHE_ERR_INTERNAL: return "internal error"; - case CACHE_ERR_REDIS_JSON: return "parse redis json error"; - case CACHE_ERR_REDIS_CONNECT:return "redis is not connected"; - case CACHE_ERR_REDIS_EXEC: return "redis command reply error"; - case CACHE_OUTOF_SESSION: return "two many curl sessions"; - case CACHE_UPDATE_CANCELED: return "update was canceled"; - case CACHE_ERR_EVBUFFER: return "evbuffer read write error"; - default: return ctx->error; - } -} - -void tango_cache_get_statistics(const struct tango_cache_instance *instance, struct cache_statistics *out) -{ - *out = instance->statistic; -} - -struct tango_cache_result *tango_cache_read_result(future_result_t *promise_result) -{ - return (struct tango_cache_result *)promise_result; -} - -static void update_statistics(struct tango_cache_ctx *ctx, struct cache_statistics *statistic) -{ - switch(ctx->method) - { - case CACHE_REQUEST_PUT: - if(ctx->fail_state) - { - if(ctx->locate == OBJECT_IN_HOS) - statistic->put_err_http += 1; - else - statistic->put_err_redis += 1; - } - else - { - if(ctx->locate == OBJECT_IN_HOS) - statistic->put_succ_http += 1; - else - statistic->put_succ_redis += 1; - } - break; - case CACHE_REQUEST_GET: - case CACHE_REQUEST_HEAD: - if(ctx->fail_state) - { - if(ctx->error_code == CACHE_CACHE_MISS || ctx->error_code == CACHE_TIMEOUT) - statistic->get_miss_num += 1; - else if(ctx->locate == OBJECT_IN_HOS) - statistic->get_err_http += 1; - else - statistic->get_err_redis += 1; - } - else - { - if(ctx->locate == OBJECT_IN_HOS) - statistic->get_succ_http += 1; - else - statistic->get_succ_redis += 1; - } - break; - case CACHE_REQUEST_DELETE: - if(ctx->fail_state) - { - statistic->del_error_num += 1; - } - else - { - statistic->del_succ_num += 1; - } - break; - case CACHE_REQUEST_DELETE_MUL: - statistic->del_succ_num += ctx->del.succ_num; - statistic->del_error_num += ctx->del.fail_num; - break; - default:break; - } -} - -void easy_string_destroy(struct easy_string *estr) -{ - if(estr->buff != NULL) - { - free(estr->buff); - estr->buff = NULL; - estr->len = estr->size = 0; - } -} - -void easy_string_savedata(struct easy_string *estr, const char *data, size_t len) -{ - if(estr->size-estr->len < len+1) - { - estr->size += len*4+1; - estr->buff = (char*)realloc(estr->buff, estr->size); - } - - memcpy(estr->buff+estr->len, data, len); - estr->len += len; - estr->buff[estr->len]='\0'; -} - -//callback: �Ƿ���ûص���������ҪΪ���ֱ�ӵ���APIʱʧ�ܣ����ٵ��ûص�������ͨ������ֵ�ж� -void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx, bool callback) -{ - struct multipart_etag_list *etag; - - if(ctx->curl != NULL) - { - curl_multi_remove_handle(ctx->instance->multi_hd, ctx->curl); - curl_easy_cleanup(ctx->curl); - } - easy_string_destroy(&ctx->response); - - switch(ctx->method) - { - case CACHE_REQUEST_GET: - case CACHE_REQUEST_HEAD: - easy_string_destroy(&ctx->get.response_tag); - break; - - case CACHE_REQUEST_PUT: - if(ctx->put.uploadID != NULL) free(ctx->put.uploadID); - if(ctx->put.combine_xml != NULL) free(ctx->put.combine_xml); - if(ctx->put.object_meta != NULL) cJSON_Delete(ctx->put.object_meta); - if(ctx->put.once_request.len > 0) - { - ctx->instance->statistic.memory_used -= ctx->put.once_request.size; - easy_string_destroy(&ctx->put.once_request); - } - if(ctx->put.evbuf!=NULL) - { - ctx->instance->statistic.memory_used -= evbuffer_get_length(ctx->put.evbuf); - evbuffer_free(ctx->put.evbuf); - } - while(NULL != (etag = TAILQ_FIRST(&ctx->put.etag_head))) - { - TAILQ_REMOVE(&ctx->put.etag_head, etag, node); - free(etag->etag); - free(etag); - }//no break here - case CACHE_REQUEST_DELETE_MUL: - if(ctx->headers != NULL) - { - curl_slist_free_all(ctx->headers); - }//no break here - case CACHE_REQUEST_DELETE: - if(callback && ctx->promise != NULL) - { - if(ctx->fail_state) - { - promise_failed(ctx->promise, FUTURE_ERROR_CANCEL, tango_cache_get_errstring(ctx)); - } - else - { - promise_success(ctx->promise, NULL); - } - } - break; - default: break; - } - update_statistics(ctx, &ctx->instance->statistic); - free(ctx); -} - -//�ж�session�Ƿ����ƣ�����ȡ��ʼ���ж�where_to_get�Ƿ�ȫ����MINIO�� -bool sessions_exceeds_limit(struct tango_cache_instance *instance, enum OBJECT_LOCATION where_to_get) -{ - if(where_to_get == OBJECT_IN_HOS) - { - return (instance->statistic.session_http>=instance->param->maximum_sessions); - } - else - { - return (instance->statistic.session_redis>=instance->param->maximum_sessions); - } -} - - -//�����ϴ�API��ʹ��ctx��evbuffer������������ctx��ȡ���� -enum OBJECT_LOCATION tango_cache_object_locate(struct tango_cache_instance *instance, size_t object_size) -{ - if(instance->param->fsstatid_trig) - { - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_histlen_id, 0, FS_OP_SET, object_size); - } - if(instance->param->object_store_way!=CACHE_SMALL_REDIS || object_size > instance->param->redis_object_maxsize) - { - return OBJECT_IN_HOS; - } - else - { - return OBJECT_IN_REDIS; - } -} - -void tango_cache_get_object_path(struct tango_cache_ctx *ctx, char *path/*OUT*/, size_t pathsize) -{ - if(path != NULL) - { - if(ctx->locate == OBJECT_IN_HOS) - { - snprintf(path, pathsize, "http://%s/%s", ctx->hostaddr, ctx->object_key); - } - else - { - snprintf(path, pathsize, "redis://%s/%s", ctx->instance->redisaddr, ctx->object_key); - } - } -} - -int tango_cache_update_end(struct tango_cache_ctx *ctx, char *path/*OUT*/, size_t pathsize) -{ - if(!ctx->fail_state) - { - ctx->locate = tango_cache_object_locate(ctx->instance, ctx->put.object_size); - tango_cache_get_object_path(ctx, path, pathsize); - - if(ctx->instance->param->object_store_way != CACHE_ALL_HOS) - { - cJSON_AddNumberToObject(ctx->put.object_meta, "Content-Length", ctx->put.object_size); - } - } - return do_tango_cache_update_end(ctx, false); -} - -void tango_cache_update_cancel(struct tango_cache_ctx *ctx) -{ - ctx->put.close_state = true; - if(ctx->curl != NULL) - { - curl_multi_remove_handle(ctx->instance->multi_hd, ctx->curl); - curl_easy_cleanup(ctx->curl); - ctx->curl = NULL; - } - tango_cache_set_fail_state(ctx, CACHE_UPDATE_CANCELED); - if(ctx->put.uploadID!=NULL && cache_cancel_upload_minio(ctx)) - { - ctx->put.state = PUT_STATE_CANCEL; - } - else - { - tango_cache_ctx_destroy(ctx); - } -} - -int tango_cache_update_frag_data(struct tango_cache_ctx *ctx, const char *data, size_t size) -{ - if(ctx->fail_state) - { - return 0; //TODO: ��ʱ���Է���ֵ!! - } - if(evbuffer_add(ctx->put.evbuf, data, size)) - { - tango_cache_set_fail_state(ctx, CACHE_OUTOF_MEMORY); - return 0; - } - ctx->instance->statistic.memory_used += size; - ctx->put.object_size += size; - if(evbuffer_get_length(ctx->put.evbuf) >= ctx->instance->param->upload_block_size) - { - cache_kick_upload_minio_multipart(ctx, ctx->instance->param->upload_block_size); - } - return 0; -} - -int tango_cache_update_frag_evbuf(struct tango_cache_ctx *ctx, enum EVBUFFER_COPY_WAY way, struct evbuffer *evbuf) -{ - size_t size; - - if(ctx->fail_state) - { - return 0;//TODO: ��ʱ���Է���ֵ!! - } - - size = evbuffer_get_length(evbuf); - if(way == EVBUFFER_MOVE) - { - if(evbuffer_add_buffer(ctx->put.evbuf, evbuf)) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_EVBUFFER); - return 0; - } - } - else - { - if(evbuffer_add_buffer_reference(ctx->put.evbuf, evbuf)) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_EVBUFFER); - return 0; - } - } - ctx->instance->statistic.memory_used += size; - ctx->put.object_size += size; - if(evbuffer_get_length(ctx->put.evbuf) >= ctx->instance->param->upload_block_size) - { - cache_kick_upload_minio_multipart(ctx, ctx->instance->param->upload_block_size); - } - return 0; -} - -struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *instance, struct future* f, struct tango_cache_meta_put *meta, enum OBJECT_LOCATION maybe_loc) -{ - struct tango_cache_ctx *ctx; - char buffer[2064], *user_tag=NULL, *user_tag_value=NULL; - time_t expires, now, last_modify; - struct easy_string hdr_estr={NULL, 0, 0}; - - if(sessions_exceeds_limit(instance, maybe_loc) || (u_int64_t)instance->statistic.memory_used>=instance->param->maximum_used_mem) - { - instance->error_code = CACHE_OUTOF_MEMORY; - instance->statistic.totaldrop_num += 1; - return NULL; - } - - ctx = (struct tango_cache_ctx *)calloc(1, sizeof(struct tango_cache_ctx)); - ctx->instance = instance; - ctx->promise = future_to_promise(f); - ctx->method = CACHE_REQUEST_PUT; - ctx->locate = maybe_loc; - - if(instance->param->hash_object_key) - { - caculate_sha256(meta->url, strlen(meta->url), buffer, 72); - if(meta->user_log_name) - { - struct timespec start_time; - clock_gettime(CLOCK_REALTIME,&start_time); - snprintf(ctx->object_key, 256, "%s/%lu_%c%c_%c%c_%s", instance->param->bucketname, start_time.tv_nsec, buffer[0], buffer[1], buffer[2], buffer[3], buffer+4); - } - else - { - snprintf(ctx->object_key, 256, "%s/%c%c_%c%c_%s", instance->param->bucketname, buffer[0], buffer[1], buffer[2], buffer[3], buffer+4); - } - //����ԭʼURL - snprintf(buffer, 2064, "x-amz-meta-url: %s", meta->url); - ctx->headers = curl_slist_append(ctx->headers, buffer); - } - else - { - snprintf(ctx->object_key, 256, "%s/%s", instance->param->bucketname, meta->url); - } - if(wired_load_balancer_lookup(instance->param->cache.wiredlb, ctx->object_key, strlen(ctx->object_key), ctx->hostaddr, 48)) - { - instance->error_code = CACHE_ERR_WIREDLB; - instance->statistic.totaldrop_num += 1; - if(ctx->headers!=NULL) curl_slist_free_all(ctx->headers); - free(ctx); - return NULL; - } - - //Expires�ֶΣ����ڻ����ڲ��ж������Ƿ�ʱ - now = time(NULL); - expires = (meta->put.timeout==0||meta->put.timeout>instance->param->relative_ttl)?instance->param->relative_ttl:meta->put.timeout; - if(expires_timestamp2hdr_str(now + expires, buffer, 256)) - { - ctx->headers = curl_slist_append(ctx->headers, buffer); - } - ctx->put.object_ttl = expires; - //Last-Modify�ֶΣ�����GETʱ�ж��Ƿ����� - last_modify = (meta->put.date > meta->put.last_modified)?meta->put.date:meta->put.last_modified; - if(last_modify == 0) - { - last_modify = get_gmtime_timestamp(now); - } - sprintf(buffer, "x-amz-meta-lm: %lu", last_modify); - ctx->headers = curl_slist_append(ctx->headers, buffer); - //�б���֧�ֵı�ͷ�� - for(int i=0; i<HDR_CONTENT_NUM; i++) - { - if(meta->std_hdr[i] != NULL) - { - ctx->headers = curl_slist_append(ctx->headers, meta->std_hdr[i]); - if(ctx->instance->param->object_store_way != CACHE_ALL_HOS) - { - easy_string_savedata(&hdr_estr, meta->std_hdr[i], strlen(meta->std_hdr[i])); - easy_string_savedata(&hdr_estr, "\r\n", 2); - } - } - } - if(meta->std_hdr[HDR_CONTENT_TYPE] == NULL) - { - ctx->headers = curl_slist_append(ctx->headers, "Content-Type:"); - if(ctx->instance->param->object_store_way != CACHE_ALL_HOS) - { - easy_string_savedata(&hdr_estr, "Content-Type: application/octet-stream\r\n", strlen("Content-Type: application/octet-stream\r\n")); - } - } - ctx->headers = curl_slist_append(ctx->headers, "Expect:");//ע��POST������Expect��ϵ��Ҫ��ȷ����CURLOPT_POSTFIELDSIZE - //���������ͷ����GETʱ��ԭ������ - if(meta->usertag_len>0 && meta->usertag_len<=USER_TAG_MAX_LEN) - { - user_tag = (char *)malloc((meta->usertag_len/3 + 1)*4 + 18); //������������ռ䣻18=17+1: ͷ��+�ַ��������� - memcpy(user_tag, "x-amz-meta-user: ", 17); - user_tag_value = user_tag+17; - Base64_EncodeBlock((const unsigned char*)meta->usertag, meta->usertag_len, (unsigned char*)user_tag_value); - ctx->headers = curl_slist_append(ctx->headers, user_tag); - } - - if(ctx->instance->param->object_store_way != CACHE_ALL_HOS) - { - ctx->put.object_meta = cJSON_CreateObject(); - if(instance->param->hash_object_key) - { - cJSON_AddStringToObject(ctx->put.object_meta, "X-Amz-Meta-Url", meta->url); - } - cJSON_AddNumberToObject(ctx->put.object_meta, "Expires", now + expires); - cJSON_AddNumberToObject(ctx->put.object_meta, "X-Amz-Meta-Lm", last_modify); - cJSON_AddStringToObject(ctx->put.object_meta, "Headers", hdr_estr.buff); - if(user_tag_value != NULL) - { - cJSON_AddStringToObject(ctx->put.object_meta, "X-Amz-Meta-User", user_tag_value); - } - easy_string_destroy(&hdr_estr); - } - if(user_tag != NULL) - { - free(user_tag); - } - - ctx->put.evbuf = evbuffer_new(); - TAILQ_INIT(&ctx->put.etag_head); - return ctx; -} - -struct tango_cache_ctx *tango_cache_update_start(struct tango_cache_instance *instance, struct future* f, struct tango_cache_meta_put *meta) -{ - struct tango_cache_ctx *ctx; - enum OBJECT_LOCATION maybe_loc=OBJECT_IN_UNKNOWN; - - if(instance->param->object_store_way != CACHE_SMALL_REDIS) - { - maybe_loc = OBJECT_IN_HOS; - } - - ctx = tango_cache_update_prepare(instance, f, meta, maybe_loc); - if(ctx == NULL) - { - return NULL; - } - ctx->instance->statistic.put_recv_num += 1; - ctx->instance->error_code = CACHE_OK; - return ctx; -} - -//һ�����ϴ�ʱ��ֱ�Ӷ�λ�����ϴ���λ�� -struct tango_cache_ctx *tango_cache_update_once_prepare(struct tango_cache_instance *instance, struct future* f, struct tango_cache_meta_put *meta, - size_t object_size, char *path, size_t pathsize) -{ - struct tango_cache_ctx *ctx; - enum OBJECT_LOCATION location; - - location = tango_cache_object_locate(instance, object_size); - ctx = tango_cache_update_prepare(instance, f, meta, location); - if(ctx == NULL) - { - return NULL; - } - tango_cache_get_object_path(ctx, path, pathsize); - - if(ctx->instance->param->object_store_way != CACHE_ALL_HOS) - { - cJSON_AddNumberToObject(ctx->put.object_meta, "Content-Length", object_size); - } - return ctx; -} - -int tango_cache_upload_once_data(struct tango_cache_instance *instance, struct future* f, - enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, struct tango_cache_meta_put *meta, char *path, size_t pathsize) -{ - struct tango_cache_ctx *ctx; - - ctx = tango_cache_update_once_prepare(instance, f, meta, size, path, pathsize); - if(ctx == NULL) - { - if(way == PUT_MEM_FREE) free((void *)data); - return -1; - } - return do_tango_cache_upload_once_data(ctx, way, data, size); -} - -int tango_cache_upload_once_evbuf(struct tango_cache_instance *instance, struct future* f, - enum EVBUFFER_COPY_WAY way, struct evbuffer *evbuf, struct tango_cache_meta_put *meta, char *path, size_t pathsize) -{ - struct tango_cache_ctx *ctx; - - ctx = tango_cache_update_once_prepare(instance, f, meta, evbuffer_get_length(evbuf), path, pathsize); - if(ctx == NULL) - { - return -1; - } - return do_tango_cache_upload_once_evbuf(ctx, way, evbuf); -} - -struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *instance, enum CACHE_REQUEST_METHOD method, - struct future* f, struct tango_cache_meta_get *meta, enum OBJECT_LOCATION where_to_get) -{ - struct tango_cache_ctx *ctx; - char sha256[72]; - - if(sessions_exceeds_limit(instance, where_to_get)) - { - instance->error_code = CACHE_OUTOF_SESSION; - instance->statistic.totaldrop_num += 1; - return NULL; - } - - ctx = (struct tango_cache_ctx *)calloc(1, sizeof(struct tango_cache_ctx)); - ctx->instance = instance; - ctx->promise = future_to_promise(f); - promise_allow_many_successes(ctx->promise); //��λص��������ʱ����promise_finish - ctx->method = method; - ctx->get.state = GET_STATE_START; - ctx->get.max_age = meta->get.max_age; - ctx->get.min_fresh = meta->get.min_fresh; - - if(instance->param->hash_object_key) - { - caculate_sha256(meta->url, strlen(meta->url), sha256, 72); - snprintf(ctx->object_key, 256, "%s/%c%c_%c%c_%s", instance->param->bucketname, sha256[0], sha256[1], sha256[2], sha256[3], sha256+4); - } - else - { - snprintf(ctx->object_key, 256, "%s/%s", instance->param->bucketname, meta->url); - } - if(wired_load_balancer_lookup(instance->param->cache.wiredlb, ctx->object_key, strlen(ctx->object_key), ctx->hostaddr, 48)) - { - instance->error_code = CACHE_ERR_WIREDLB; - instance->statistic.totaldrop_num += 1; - free(ctx); - return NULL; - } - return ctx; -} - -int tango_cache_fetch_object(struct tango_cache_instance *instance, struct future* f, struct tango_cache_meta_get *meta, enum OBJECT_LOCATION where_to_get) -{ - struct tango_cache_ctx *ctx; - - if(instance->param->object_store_way != CACHE_SMALL_REDIS) - { - where_to_get = OBJECT_IN_HOS; - } - - ctx = tango_cache_fetch_prepare(instance, CACHE_REQUEST_GET, f, meta, where_to_get); - if(ctx == NULL) - { - return -1; - } - return do_tango_cache_fetch_object(ctx, where_to_get); -} - -int tango_cache_head_object(struct tango_cache_instance *instance, struct future* f, struct tango_cache_meta_get *meta) -{ - struct tango_cache_ctx *ctx; - enum OBJECT_LOCATION location; - - //���������Redis����Ԫ��Ϣ�洢��Redis�� - location = (instance->param->object_store_way != CACHE_ALL_HOS)?OBJECT_IN_REDIS:OBJECT_IN_HOS; - ctx = tango_cache_fetch_prepare(instance, CACHE_REQUEST_HEAD, f, meta, location); - if(ctx == NULL) - { - return -1; - } - return do_tango_cache_head_object(ctx, location); -} - -struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *instance, struct future* f, const char *objkey, const char *minio_addr, const char *bucket) -{ - struct tango_cache_ctx *ctx; - char sha256[72]; - const char *pbucket; - - if(sessions_exceeds_limit(instance, OBJECT_IN_HOS)) - { - instance->error_code = CACHE_OUTOF_SESSION; - instance->statistic.totaldrop_num += 1; - return NULL; - } - - ctx = (struct tango_cache_ctx *)calloc(1, sizeof(struct tango_cache_ctx)); - ctx->instance = instance; - ctx->promise = future_to_promise(f); - ctx->method = CACHE_REQUEST_DELETE; - - pbucket = (bucket==NULL)?instance->param->bucketname:bucket; - if(instance->param->hash_object_key) - { - caculate_sha256(objkey, strlen(objkey), sha256, 72); - snprintf(ctx->object_key, 256, "%s/%c%c_%c%c_%s", pbucket, sha256[0], sha256[1], sha256[2], sha256[3], sha256+4); - } - else - { - snprintf(ctx->object_key, 256, "%s/%s", pbucket, objkey); - } - if(minio_addr != NULL) - { - snprintf(ctx->hostaddr, 48, "%s", minio_addr); - } - else if(wired_load_balancer_lookup(instance->param->cache.wiredlb, ctx->object_key, strlen(ctx->object_key), ctx->hostaddr, 48)) - { - instance->error_code = CACHE_ERR_WIREDLB; - instance->statistic.totaldrop_num += 1; - free(ctx); - return NULL; - } - return ctx; -} - -int tango_cache_delete_object(struct tango_cache_instance *instance, struct future* f, const char *objkey, const char *minio_addr, const char *bucket) -{ - struct tango_cache_ctx *ctx; - - ctx = tango_cache_delete_prepare(instance, f, objkey, minio_addr, bucket); - if(ctx == NULL) - { - return -1; - } - return (cache_delete_minio_object(ctx)==1)?0:-1; -} - -struct tango_cache_ctx *tango_cache_multi_delete_prepare(struct tango_cache_instance *instance, struct future* f, char *objlist[], u_int32_t num) -{ - struct tango_cache_ctx *ctx; - char md5[48]={0}, content_md5[64]; - - if(sessions_exceeds_limit(instance, OBJECT_IN_HOS)) - { - instance->error_code = CACHE_OUTOF_SESSION; - instance->statistic.totaldrop_num += 1; - return NULL; - } - - ctx = (struct tango_cache_ctx *)calloc(1, sizeof(struct tango_cache_ctx)); - ctx->instance = instance; - ctx->promise = future_to_promise(f); - ctx->method = CACHE_REQUEST_DELETE_MUL; - ctx->del.succ_num = num; - - if(wired_load_balancer_lookup(instance->param->cache.wiredlb, objlist[0], strlen(objlist[0]), ctx->hostaddr, 48)) - { - instance->error_code = CACHE_ERR_WIREDLB; - instance->statistic.totaldrop_num += num; - free(ctx); - return NULL; - } - - construct_multiple_delete_xml(instance->param->bucketname, objlist, num, instance->param->hash_object_key, &ctx->response.buff, &ctx->response.size); - caculate_base64_md5(ctx->response.buff, ctx->response.size, (unsigned char *)md5, 48); - sprintf(content_md5, "Content-MD5: %s", md5); - ctx->headers = curl_slist_append(ctx->headers, content_md5); - ctx->headers = curl_slist_append(ctx->headers, "Content-Type: application/xml"); - ctx->headers = curl_slist_append(ctx->headers, "Expect:"); - return ctx; -} - -//TODO: AccessDenied -int tango_cache_multi_delete(struct tango_cache_instance *instance, struct future* f, char *objlist[], u_int32_t num) -{ - struct tango_cache_ctx *ctx; - - ctx = tango_cache_multi_delete_prepare(instance, f, objlist, num); - if(ctx == NULL) - { - return -1; - } - return do_tango_cache_multi_delete(ctx); -} - -static void check_multi_info(CURLM *multi) -{ - CURLMsg *msg; - int msgs_left; - struct tango_cache_ctx *ctx; - CURL *easy; - CURLcode res; - long res_code; - - while((msg = curl_multi_info_read(multi, &msgs_left))) - { - if(msg->msg != CURLMSG_DONE) - { - continue; - } - - easy = msg->easy_handle; - res = msg->data.result; - curl_easy_getinfo(easy, CURLINFO_PRIVATE, &ctx); - curl_easy_getinfo(easy, CURLINFO_RESPONSE_CODE, &res_code); - curl_multi_remove_handle(multi, easy); - curl_easy_cleanup(easy); - ctx->curl = NULL; - ctx->res_code = 0; - - switch(ctx->method) - { - case CACHE_REQUEST_GET: - case CACHE_REQUEST_HEAD: - tango_cache_curl_get_done(ctx, res, res_code); - break; - case CACHE_REQUEST_PUT: - tango_cache_curl_put_done(ctx, res, res_code); - break; - case CACHE_REQUEST_DELETE: - tango_cache_curl_del_done(ctx, res, res_code); - break; - case CACHE_REQUEST_DELETE_MUL: - tango_cache_curl_muldel_done(ctx, res, res_code); - break; - default: break; - } - } -} - -/* Called by libevent when we get action on a multi socket */ -static void libevent_socket_event_cb(int fd, short action, void *userp) -{ - struct tango_cache_instance *instance = (struct tango_cache_instance *)userp; //from event_assign - UNUSED CURLMcode rc; - int what, still_running; - - what = ((action&EV_READ)?CURL_CSELECT_IN:0) | ((action & EV_WRITE)?CURL_CSELECT_OUT:0); - - rc = curl_multi_socket_action(instance->multi_hd, fd, what, &still_running); - instance->statistic.session_http = still_running; - assert(rc==CURLM_OK); - - check_multi_info(instance->multi_hd); - if(still_running<=0 && evtimer_pending(&instance->timer_event, NULL)) - { - evtimer_del(&instance->timer_event); - } -} - -/* Called by libevent when our timeout expires */ -static void libevent_timer_event_cb(int fd, short kind, void *userp) -{ - struct tango_cache_instance *instance = (struct tango_cache_instance *)userp; - UNUSED CURLMcode rc; - int still_running; - - rc = curl_multi_socket_action(instance->multi_hd, CURL_SOCKET_TIMEOUT, 0, &still_running); - instance->statistic.session_http = still_running; - assert(rc==CURLM_OK); - check_multi_info(instance->multi_hd); -} - -static int curl_socket_function_cb(CURL *curl, curl_socket_t sockfd, int what, void *userp, void *sockp) -{ - struct tango_cache_instance *instance = (struct tango_cache_instance *)userp; //from multi handle - struct curl_socket_data *sockinfo = (struct curl_socket_data *)sockp; //curl_multi_assign, for socket - int action; - - if(what == CURL_POLL_REMOVE) - { - if(sockinfo != NULL) - { - event_del(&sockinfo->sock_event); - free(sockinfo); - } - } - else - { - if(sockinfo == NULL) - { - sockinfo = (struct curl_socket_data *)calloc(1, sizeof(struct curl_socket_data)); - curl_multi_assign(instance->multi_hd, sockfd, sockinfo); - } - else - { - event_del(&sockinfo->sock_event); - } - - action = (what&CURL_POLL_IN?EV_READ:0)|(what&CURL_POLL_OUT?EV_WRITE:0)|EV_PERSIST; - event_assign(&sockinfo->sock_event, instance->evbase, sockfd, action, libevent_socket_event_cb, instance); - event_add(&sockinfo->sock_event, NULL); - } - - return 0; -} - -static int curl_timer_function_cb(CURLM *multi, long timeout_ms, void *userp) -{ - struct tango_cache_instance *instance = (struct tango_cache_instance *)userp; - struct timeval timeout; - UNUSED CURLMcode rc; - int still_running; - - timeout.tv_sec = timeout_ms/1000; - timeout.tv_usec = (timeout_ms%1000)*1000; - - if(timeout_ms == 0) - { - //timeout_ms is 0 means we should call curl_multi_socket_action/curl_multi_perform at once. - //To initiate the whole process(inform CURLMOPT_SOCKETFUNCTION callback) or when timeout occurs. - rc = curl_multi_socket_action(multi, CURL_SOCKET_TIMEOUT, 0, &still_running); - instance->statistic.session_http = still_running; - assert(rc==CURLM_OK); - } - else if(timeout_ms == -1) //timeout_ms is -1 means we should delete the timer. - { - evtimer_del(&instance->timer_event); - } - else //update the timer to the new value. - { - evtimer_add(&instance->timer_event, &timeout); - } - - return 0; //0-success; -1-error -} - -static void instance_statistic_timer_cb(int fd, short kind, void *userp) -{ - struct tango_cache_instance *instance = (struct tango_cache_instance *)userp; - struct timeval tv; - struct cache_statistics incr_statistic; - long long *plast_statistic = (long long*)&instance->statistic_last; - long long *pnow_statistic = (long long*)&instance->statistic; - long long *pinc_statistic = (long long*)&incr_statistic; - - for(u_int32_t i=0; i<sizeof(struct cache_statistics)/sizeof(long long); i++) - { - pinc_statistic[i] = pnow_statistic[i] - plast_statistic[i]; - } - instance->statistic_last = instance->statistic; - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_GET_RECV], 0, FS_OP_ADD, incr_statistic.get_recv_num); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_GET_S_TOTAL], 0, FS_OP_ADD, incr_statistic.get_succ_http+incr_statistic.get_succ_redis); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_GET_S_HTTP], 0, FS_OP_ADD, incr_statistic.get_succ_http); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_GET_S_REDIS], 0, FS_OP_ADD, incr_statistic.get_succ_redis); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_GET_MISS], 0, FS_OP_ADD, incr_statistic.get_miss_num); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_GET_E_TOTAL], 0, FS_OP_ADD, incr_statistic.get_err_http+incr_statistic.get_err_redis); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_GET_E_HTTP], 0, FS_OP_ADD, incr_statistic.get_err_http); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_GET_E_REDIS], 0, FS_OP_ADD, incr_statistic.get_err_redis); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_PUT_RECV], 0, FS_OP_ADD, incr_statistic.put_recv_num); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_PUT_S_TOTAL], 0, FS_OP_ADD, incr_statistic.put_succ_http+incr_statistic.put_succ_redis); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_PUT_S_HTTP], 0, FS_OP_ADD, incr_statistic.put_succ_http); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_PUT_S_REDIS], 0, FS_OP_ADD, incr_statistic.put_succ_redis); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_PUT_E_TOTAL], 0, FS_OP_ADD, incr_statistic.put_err_http+incr_statistic.put_err_redis); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_PUT_E_HTTP], 0, FS_OP_ADD, incr_statistic.put_err_http); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_PUT_E_REDIS], 0, FS_OP_ADD, incr_statistic.put_err_redis); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_DEL_RECV], 0, FS_OP_ADD, incr_statistic.del_recv_num); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_DEL_SUCC], 0, FS_OP_ADD, incr_statistic.del_succ_num); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_DEL_ERROR], 0, FS_OP_ADD, incr_statistic.del_error_num); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_TOTAL_DROP], 0, FS_OP_ADD, incr_statistic.totaldrop_num); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_MEM_USED], 0, FS_OP_ADD, incr_statistic.memory_used); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_SESS_HTTP], 0, FS_OP_ADD, incr_statistic.session_http); - FS_operate(instance->param->fsstat_handle, instance->param->fsstat_field_ids[FS_FILED_SESS_REDIS], 0, FS_OP_ADD, incr_statistic.session_redis); - tv.tv_sec = instance->param->fsstat_period; - tv.tv_usec = 0; - event_add(&instance->timer_statistic, &tv); -} - -static int _unfold_IP_range(char* ip_range, char***ip_list, int size) -{ - int i=0,count=0, ret=0; - int range_digits[5]; - memset(range_digits,0,sizeof(range_digits)); - ret=sscanf(ip_range,"%d.%d.%d.%d-%d",&range_digits[0],&range_digits[1],&range_digits[2],&range_digits[3],&range_digits[4]); - if(ret!=4&&ret!=5) - { - return 0; - } - if(ret==4&&range_digits[4]==0) - { - range_digits[4]=range_digits[3]; - } - for(i=0;i<5;i++) - { - if(range_digits[i]<0||range_digits[i]>255) - { - return 0; - } - } - count=range_digits[4]-range_digits[3]+1; - *ip_list=(char**)realloc(*ip_list, sizeof(char*)*(size+count)); - for(i=0;i<count;i++) - { - (*ip_list)[size+i]=(char*)malloc(64); - snprintf((*ip_list)[size+i],64,"%d.%d.%d.%d",range_digits[0],range_digits[1],range_digits[2],range_digits[3]+i); - } - return count; -} - -static int unfold_IP_range(const char* ip_range, char***ip_list) -{ - char *token=NULL,*sub_token=NULL,*saveptr; - char *buffer=(char*)calloc(sizeof(char),strlen(ip_range)+1); - int count=0; - strcpy(buffer,ip_range); - for (token = buffer; ; token= NULL) - { - sub_token= strtok_r(token,";", &saveptr); - if (sub_token == NULL) - break; - count+=_unfold_IP_range(sub_token, ip_list,count); - } - free(buffer); - return count; -} - -static int build_redis_cluster_addrs(const char *iplist, const char *ports, char *redisaddrs, size_t size, void *runtimelog) -{ - u_int32_t redis_ip_num; - u_int32_t redis_port_start, redis_port_end; - char **redis_iplist=NULL; - size_t addrlen; - int ret; - - redis_ip_num = unfold_IP_range(iplist, &redis_iplist); - if(redis_ip_num ==0 ) - { - MESA_HANDLE_RUNTIME_LOGV2(runtimelog, RLOG_LV_FATAL, "decode REDIS_CLUSTER_IP_LIST %s failed.", iplist); - return -1; - } - ret = sscanf(ports, "%u-%u", &redis_port_start, &redis_port_end); - if(ret!=1 && ret!=2) - { - MESA_HANDLE_RUNTIME_LOGV2(runtimelog, RLOG_LV_FATAL, "decode REDIS_CLUSTER_PORT_RANGE %s failed.", iplist); - return -2; - } - - memset(redisaddrs, 0, size); - for(u_int32_t i=0; i<redis_ip_num; i++) - { - addrlen = strlen(redisaddrs); - snprintf(redisaddrs+addrlen, size-addrlen, "%s:%u,", redis_iplist[i], redis_port_start); - } - addrlen = strlen(redisaddrs); - redisaddrs[addrlen-1] = '\0'; - return 0; -} - -static int wired_load_balancer_init(struct wiredlb_parameter *wparam, void *runtime_log) -{ - wparam->wiredlb = wiredLB_create(wparam->wiredlb_topic, wparam->wiredlb_group, WLB_PRODUCER); - if(wparam->wiredlb == NULL) - { - MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "wiredLB_create failed.\n"); - return -1; - } - wiredLB_set_opt(wparam->wiredlb, WLB_OPT_HEALTH_CHECK_PORT, &wparam->wiredlb_ha_port, sizeof(wparam->wiredlb_ha_port)); - wiredLB_set_opt(wparam->wiredlb, WLB_OPT_ENABLE_OVERRIDE, &wparam->wiredlb_override, sizeof(wparam->wiredlb_override)); - if(strlen(wparam->wiredlb_datacenter) > 0) - { - wiredLB_set_opt(wparam->wiredlb, WLB_PROD_OPT_DATACENTER, wparam->wiredlb_datacenter, strlen(wparam->wiredlb_datacenter)+1); - } - if(wparam->wiredlb_override) - { - wiredLB_set_opt(wparam->wiredlb, WLB_PROD_OPT_OVERRIDE_PRIMARY_IP, wparam->iplist, strlen(wparam->iplist)+1); - wiredLB_set_opt(wparam->wiredlb, WLB_PROD_OPT_OVERRIDE_DATAPORT, &wparam->port, sizeof(wparam->port)); - } - if(wiredLB_init(wparam->wiredlb) < 0) - { - MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "wiredLB_init group %s failed.\n", wparam->wiredlb_group); - return -1; - } - return 0; -} - -int register_field_stat(struct tango_cache_parameter *param, void *runtime_log) -{ - int value; - const char *field_names[FS_FILED_NUM]={"GET_RECV", "GET_S_TOTAL", "GET_S_HTTP", "GET_S_REDIS", "GET_MISS", "GET_E_TOTAL", "GET_E_HTTP", "GET_E_REDIS", - "PUT_RECV", "PUT_S_TOTAL", "PUT_S_HTTP", "PUT_S_REDIS", "PUT_E_TOTAL", "PUT_E_HTTP", "PUT_E_REDIS", - "DEL_RECV", "DEL_SUCC", "DEL_ERROR", "TOTAL_DROP", "MEM_USED", "SESSION_HTTP", "SESSION_REDIS"}; - - param->fsstat_handle = FS_create_handle(); - FS_set_para(param->fsstat_handle, OUTPUT_DEVICE, param->fsstat_filepath, strlen(param->fsstat_filepath)+1); - value = 1; - FS_set_para(param->fsstat_handle, PRINT_MODE, &value, sizeof(value)); - value = 2; - FS_set_para(param->fsstat_handle, STAT_CYCLE, &value, sizeof(value)); - value = 1; - FS_set_para(param->fsstat_handle, CREATE_THREAD, &value, sizeof(value)); - FS_set_para(param->fsstat_handle, APP_NAME, param->fsstat_appname, strlen(param->fsstat_appname)+1); - FS_set_para(param->fsstat_handle, STATS_SERVER_IP, param->fsstat_dst_ip, strlen(param->fsstat_dst_ip)+1); - FS_set_para(param->fsstat_handle, STATS_SERVER_PORT, ¶m->fsstat_dst_port, sizeof(param->fsstat_dst_port)); - if(strlen(param->fsstat_histlen)>0 && FS_set_para(param->fsstat_handle, HISTOGRAM_GLOBAL_BINS, param->fsstat_histlen, strlen(param->fsstat_histlen)+1) < 0) - { - MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "FS_set_para %s failed.", param->fsstat_histlen); - return -1; - } - - for(int i=0; i<=FS_FILED_TOTAL_DROP; i++) - { - param->fsstat_field_ids[i] = FS_register(param->fsstat_handle, FS_STYLE_FIELD, FS_CALC_CURRENT, field_names[i]); - } - for(int i=FS_FILED_MEM_USED; i<=FS_FILED_SESS_REDIS; i++) - { - param->fsstat_field_ids[i] = FS_register(param->fsstat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT, field_names[i]); - } - param->fsstat_histlen_id = FS_register_histogram(param->fsstat_handle, FS_CALC_CURRENT, "length(bytes)", 1L, 17179869184L, 3); - if(param->fsstat_histlen_id < 0) - { - MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "FS_register_histogram failed."); - return -1; - } - FS_start(param->fsstat_handle); - return 0; -} - -struct tango_cache_parameter *tango_cache_parameter_new(const char* profile_path, const char* section, void *runtime_log) -{ - u_int32_t intval; - u_int64_t longval; - struct tango_cache_parameter *param; - char redis_cluster_ip[512], redis_ports[256]; - - param = (struct tango_cache_parameter *)calloc(1, sizeof(struct tango_cache_parameter)); - - //multi curl - MESA_load_profile_uint_def(profile_path, section, "MAX_CONNECTION_PER_HOST", &intval, 1); - param->maximum_host_cnns = intval; - MESA_load_profile_uint_def(profile_path, section, "MAX_CNNT_PIPELINE_NUM", &intval, 20); - param->maximum_pipelines = intval; - MESA_load_profile_uint_def(profile_path, section, "MAX_CURL_TRANSFER_TIMEOUT_S", &intval, 0); - param->transfer_timeout = intval; - - //instance - MESA_load_profile_uint_def(profile_path, section, "MAX_CURL_SESSION_NUM", ¶m->maximum_sessions, 100); - MESA_load_profile_uint_def(profile_path, section, "MAX_USED_MEMORY_SIZE_MB", &intval, 5120); - longval = intval; - param->maximum_used_mem = longval * 1024 * 1024; - MESA_load_profile_uint_def(profile_path, section, "CACHE_OBJECT_KEY_HASH_SWITCH", ¶m->hash_object_key, 1); - MESA_load_profile_string_def(profile_path, section, "CACHE_TOKEN", param->cache_token, 256, "c21f969b5f03d33d43e04f8f136e7682"); - if(MESA_load_profile_string_nodef(profile_path, section, "CACHE_BUCKET_NAME", param->bucketname, 256) < 0) - { - MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_BUCKET_NAME not found.\n", profile_path, section); - return NULL; - } - MESA_load_profile_uint_def(profile_path, section, "CACHE_UPLOAD_BLOCK_SIZE", ¶m->upload_block_size, 5242880); - if(param->upload_block_size < 5242880) - { - MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_UPLOAD_BLOCK_SIZE too small, must bigger than 5242880(5MB).\n", profile_path, section); - return NULL; - } - MESA_load_profile_uint_def(profile_path, section, "CACHE_DEFAULT_TTL_SECOND", &intval, 604800); - if(intval < 60) - { - MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_DEFAULT_TTL_SECOND too small, must bigger than 60s.\n", profile_path, section); - return NULL; - } - param->relative_ttl = intval; - - //wiredlb - MESA_load_profile_string_def(profile_path, section, "WIREDLB_TOPIC", param->cache.wiredlb_topic, 64, "TANGO_CACHE_PRODUCER"); - MESA_load_profile_string_nodef(profile_path, section, "WIREDLB_DATACENTER", param->cache.wiredlb_datacenter, 64); - MESA_load_profile_uint_def(profile_path, section, "WIREDLB_OVERRIDE", ¶m->cache.wiredlb_override, 1); - MESA_load_profile_uint_def(profile_path, section, "WIREDLB_HEALTH_PORT", &intval, 52100); - param->cache.wiredlb_ha_port = intval; - MESA_load_profile_string_def(profile_path, section, "WIREDLB_GROUP", param->cache.wiredlb_group, 64, "MINIO_GROUP"); - MESA_load_profile_uint_def(profile_path, section, "CACHE_LISTEN_PORT", ¶m->cache.port, 9000); - if(MESA_load_profile_string_nodef(profile_path, section, "CACHE_IP_LIST", param->cache.iplist, 4096) < 0) - { - MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "Load config %s [%s] CACHE_BROKERS_LIST not found.", profile_path, section); - return NULL; - } - if(wired_load_balancer_init(¶m->cache, runtime_log)) - { - return NULL; - } - - MESA_load_profile_int_def(profile_path, section, "CACHE_STORE_OBJECT_WAY", ¶m->object_store_way, CACHE_ALL_HOS); - if(param->object_store_way!=CACHE_ALL_HOS && param->object_store_way!=CACHE_META_REDIS && param->object_store_way!=CACHE_SMALL_REDIS) - { - MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "CACHE_STORE_OBJECT_WAY is not 1/2/3.", profile_path, section); - return NULL; - } - if(param->object_store_way != CACHE_ALL_HOS) - { - if(MESA_load_profile_string_nodef(profile_path, section, "REDIS_CLUSTER_IP_LIST", redis_cluster_ip, 512) < 0) - { - MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "Load config %s [%s] REDIS_CLUSTER_IP_LIST not found.", profile_path, section); - return NULL; - } - if(MESA_load_profile_string_nodef(profile_path, section, "REDIS_CLUSTER_PORT_RANGE", redis_ports, 256) < 0) - { - MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "Load config %s [%s] REDIS_CLUSTER_PORT_RANGE not found.", profile_path, section); - return NULL; - } - if(build_redis_cluster_addrs(redis_cluster_ip, redis_ports, param->redisaddrs, 4096, runtime_log)) - { - return NULL; - } - MESA_load_profile_uint_def(profile_path, section, "REDIS_CACHE_OBJECT_SIZE", ¶m->redis_object_maxsize, 10240); - if(param->redis_object_maxsize >= param->upload_block_size) - { - MESA_HANDLE_RUNTIME_LOGV2(runtime_log, RLOG_LV_FATAL, "Load config %s [%s] REDIS_CACHE_OBJECT_SIZE must be smaller than CACHE_UPLOAD_BLOCK_SIZE.", profile_path, section); - return NULL; - } - } - - //FieldStat LOG - MESA_load_profile_string_def(profile_path, section, "LOG_FSSTAT_APPNAME", param->fsstat_appname, 16, "TANGO_CACHE"); - MESA_load_profile_string_def(profile_path, section, "LOG_FSSTAT_FILEPATH", param->fsstat_filepath, 256, "./log/tangocache_fsstat.log"); - MESA_load_profile_uint_def(profile_path, section, "LOG_FSSTAT_INTERVAL", ¶m->fsstat_period, 10); - MESA_load_profile_uint_def(profile_path, section, "LOG_FSSTAT_TRIG", ¶m->fsstatid_trig, 0); - MESA_load_profile_string_def(profile_path, section, "LOG_FSSTAT_DST_IP", param->fsstat_dst_ip, 64, "10.172.128.2"); - MESA_load_profile_int_def(profile_path, section, "LOG_FSSTAT_DST_PORT", ¶m->fsstat_dst_port, 8125); - MESA_load_profile_string_nodef(profile_path, section, "LOG_FSSTAT_HISTBINS", param->fsstat_histlen, 256); - if(param->fsstatid_trig && register_field_stat(param, runtime_log)) - { - return NULL; - } - return param; -} - -struct tango_cache_instance *tango_cache_instance_new(struct tango_cache_parameter *param, struct event_base* evbase, void *runtimelog) -{ - struct tango_cache_instance *instance; - char *redis_sep, *save_ptr=NULL; - struct timeval tv; - time_t now, remain; - - instance = (struct tango_cache_instance *)malloc(sizeof(struct tango_cache_instance)); - memset(instance, 0, sizeof(struct tango_cache_instance)); - instance->runtime_log = runtimelog; - instance->evbase = evbase; - instance->param = param; - - instance->multi_hd = curl_multi_init(); - curl_multi_setopt(instance->multi_hd, CURLMOPT_PIPELINING, CURLPIPE_HTTP1 | CURLPIPE_MULTIPLEX); - curl_multi_setopt(instance->multi_hd, CURLMOPT_MAX_HOST_CONNECTIONS, param->maximum_host_cnns); - curl_multi_setopt(instance->multi_hd, CURLMOPT_MAX_PIPELINE_LENGTH, param->maximum_pipelines); - curl_multi_setopt(instance->multi_hd, CURLMOPT_SOCKETFUNCTION, curl_socket_function_cb); - curl_multi_setopt(instance->multi_hd, CURLMOPT_SOCKETDATA, instance); //curl_socket_function_cb *userp - curl_multi_setopt(instance->multi_hd, CURLMOPT_TIMERFUNCTION, curl_timer_function_cb); - curl_multi_setopt(instance->multi_hd, CURLMOPT_TIMERDATA, instance); - - if(param->object_store_way != CACHE_ALL_HOS) - { - if(redis_asyn_connect_init(instance)) - { - MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "redis_asyn_connect_init %s failed.", instance->param->redisaddrs); - free(instance); - return NULL; - } - redis_sep = strtok_r(param->redisaddrs, ",", &save_ptr); - sprintf(instance->redisaddr, "%s", redis_sep); - } - evtimer_assign(&instance->timer_event, evbase, libevent_timer_event_cb, instance); - - if(param->fsstatid_trig) - { - evtimer_assign(&instance->timer_statistic, evbase, instance_statistic_timer_cb, instance); - now = time(NULL); - remain = instance->param->fsstat_period - (now % instance->param->fsstat_period); - tv.tv_sec = remain; - tv.tv_usec = 0; - evtimer_add(&instance->timer_statistic, &tv); - } - return instance; -} - -void tango_cache_global_init(void) -{ - curl_global_init(CURL_GLOBAL_NOTHING); -} - diff --git a/cache/src/tango_cache_client_in.h b/cache/src/tango_cache_client_in.h deleted file mode 100644 index 82345de..0000000 --- a/cache/src/tango_cache_client_in.h +++ /dev/null @@ -1,255 +0,0 @@ -#ifndef __TANGO_CACHE_CLIENT_IN_H__ -#define __TANGO_CACHE_CLIENT_IN_H__ - -#include <curl/curl.h> -#include <sys/queue.h> -#include <pthread.h> - -#include <event2/event.h> -#include <event.h> -#include <hiredis-vip/async.h> -#include <hiredis-vip/hircluster.h> -#include <cjson/cJSON.h> - -#include <MESA/wiredLB.h> -#include <MESA/field_stat2.h> -#include "tango_cache_client.h" - -#define RESPONSE_HDR_EXPIRES 1 -#define RESPONSE_HDR_LAST_MOD 2 -#define RESPONSE_HDR_ALL 3 - -#define CACHE_ALL_HOS 0 //Ԫ��Ϣ�Ͷ�����MINIO -#define CACHE_META_REDIS 1 //Ԫ��Ϣ��REDIS������MINIO -#define CACHE_SMALL_REDIS 2 //Ԫ��Ϣ��С�ļ���REDIS�����ļ���MINIO - -enum FIELD_STAT_FILEDS -{ - FS_FILED_GET_RECV=0, - FS_FILED_GET_S_TOTAL, - FS_FILED_GET_S_HTTP, - FS_FILED_GET_S_REDIS, - FS_FILED_GET_MISS, - FS_FILED_GET_E_TOTAL, - FS_FILED_GET_E_HTTP, - FS_FILED_GET_E_REDIS, - FS_FILED_PUT_RECV, - FS_FILED_PUT_S_TOTAL, - FS_FILED_PUT_S_HTTP, - FS_FILED_PUT_S_REDIS, - FS_FILED_PUT_E_TOTAL, - FS_FILED_PUT_E_HTTP, - FS_FILED_PUT_E_REDIS, - FS_FILED_DEL_RECV, - FS_FILED_DEL_SUCC, - FS_FILED_DEL_ERROR, - FS_FILED_TOTAL_DROP, - - //Next use Status - FS_FILED_MEM_USED, - FS_FILED_SESS_HTTP, - FS_FILED_SESS_REDIS, - - FS_FILED_NUM, -}; - -enum CACHE_REQUEST_METHOD -{ - CACHE_REQUEST_GET=0, - CACHE_REQUEST_PUT, - CACHE_REQUEST_DELETE, - CACHE_REQUEST_DELETE_MUL, - CACHE_REQUEST_HEAD, -}; - -enum GET_OBJECT_STATE -{ - GET_STATE_START=0, - GET_STATE_DELETE, - GET_STATE_REDIS_META, - GET_STATE_REDIS_ALL, - GET_STATE_REDIS_TRY, - GET_STATE_END, -}; - -enum PUT_OBJECT_STATE -{ - PUT_STATE_START=0, - PUT_STATE_WAIT_START, - PUT_STATE_PART, - PUT_STATE_CANCEL, - PUT_STATE_REDIS_META, - PUT_STATE_REDIS_EXPIRE, - PUT_STATE_REDIS_SETEX, //��״̬���ڵȴ�����ִ�н�� - PUT_STATE_END, -}; - -struct easy_string -{ - char* buff; - size_t len; - size_t size; -}; - -struct wiredlb_parameter -{ - char wiredlb_topic[64]; - char wiredlb_datacenter[64]; - char wiredlb_group[64]; - char iplist[4096];//minio�б� - WLB_handle_t wiredlb; - u_int32_t wiredlb_override; - u_int32_t port; - short wiredlb_ha_port; -}; - -struct tango_cache_parameter -{ - char bucketname[256]; - char cache_token[256]; - char redis_key[256]; - long maximum_host_cnns; - long transfer_timeout;//������ʱ������ - long maximum_pipelines; - u_int64_t maximum_used_mem; - u_int32_t maximum_sessions; - u_int32_t upload_block_size; //minio�ֶ��ϴ������С���� - time_t relative_ttl; //����������Ч�� - u_int32_t hash_object_key; - //wiredlb - int object_store_way; //��ȡobject��Ϣ�ķ�ʽ - struct wiredlb_parameter cache; - char redisaddrs[4096]; - u_int32_t redis_object_maxsize;//С�ļ�����redisʱ�����������С - - //FieldStatLog - int32_t fsstat_dst_port; - char fsstat_dst_ip[64]; - char fsstat_appname[16]; - char fsstat_filepath[256]; - u_int32_t fsstat_period; - u_int32_t fsstatid_trig; - char fsstat_histlen[256]; - screen_stat_handle_t fsstat_handle; - int32_t fsstat_histlen_id; - int32_t fsstat_field_ids[FS_FILED_NUM]; -}; - -struct tango_cache_instance -{ - struct event_base* evbase; - struct event timer_event; - struct event timer_statistic; - CURLM *multi_hd; - enum CACHE_ERR_CODE error_code; - - int redis_connecting; - redisClusterAsyncContext *redis_ac; - char redisaddr[128]; - - const struct tango_cache_parameter *param; - void *runtime_log; - struct cache_statistics statistic; - struct cache_statistics statistic_last; //���ڶ��instanceʹ��ͬһ��fieldstat�ۼ� -}; - -struct multipart_etag_list -{ - char *etag; - u_int32_t part_number; - TAILQ_ENTRY(multipart_etag_list) node; -}; - -typedef void (redisRedirectMinioCallback)(struct tango_cache_ctx *ctx); - -struct cache_ctx_data_get -{ - time_t max_age; - time_t min_fresh; - time_t expires; - time_t last_modify; - u_int32_t need_hdrs; - enum GET_OBJECT_STATE state; - struct easy_string response_tag; - struct tango_cache_result result; - redisRedirectMinioCallback *redis_redirect_minio_cb; -}; - -struct cache_ctx_data_put -{ - struct evbuffer *evbuf; - size_t upload_length; - size_t upload_offset; - char *uploadID; - char *combine_xml; - TAILQ_HEAD(__etag_list_head, multipart_etag_list) etag_head; - cJSON *object_meta; - struct easy_string once_request; //һ����PUTʱ�洢�����ݣ�ʧ�ܵ�ʱ��������������ܸ��������ṹ - enum PUT_OBJECT_STATE state; - u_int32_t part_index; //��RESPONSE_HDR_ - u_int32_t object_ttl; - bool close_state; //���������ùر� - size_t object_size; -}; - -struct cache_ctx_multi_delete -{ - u_int32_t succ_num; - u_int32_t fail_num; -}; - -struct tango_cache_ctx -{ - CURL *curl; - struct curl_slist *headers; - struct promise* promise; - char error[CURL_ERROR_SIZE]; - char object_key[256]; - char hostaddr[48]; - - enum CACHE_REQUEST_METHOD method; - enum CACHE_ERR_CODE error_code; - struct easy_string response; - - bool fail_state; - enum OBJECT_LOCATION locate; //�ɳ��������϶�����λ�� - long res_code; - - union{ - struct cache_ctx_data_put put; - struct cache_ctx_data_get get; - struct cache_ctx_multi_delete del; - }; - struct tango_cache_instance *instance; -}; - -struct curl_socket_data -{ - struct event sock_event; -}; - -void caculate_sha256(const char *data, unsigned long len, char *result, u_int32_t size); - -void easy_string_savedata(struct easy_string *estr, const char *data, size_t len); -void easy_string_destroy(struct easy_string *estr); - -void tango_cache_ctx_destroy(struct tango_cache_ctx *ctx, bool callback=true); -void tango_cache_set_fail_state(struct tango_cache_ctx *ctx, enum CACHE_ERR_CODE error_code); -const char *tango_cache_get_errstring(const struct tango_cache_ctx *ctx); - -bool sessions_exceeds_limit(struct tango_cache_instance *instance, enum OBJECT_LOCATION where_to_get); - -struct tango_cache_ctx *tango_cache_update_prepare(struct tango_cache_instance *instance, - struct future* f, struct tango_cache_meta_put *meta, enum OBJECT_LOCATION maybe_loc); -struct tango_cache_ctx *tango_cache_fetch_prepare(struct tango_cache_instance *instance, - enum CACHE_REQUEST_METHOD method, struct future* f, struct tango_cache_meta_get *meta, enum OBJECT_LOCATION where_to_get); -struct tango_cache_ctx *tango_cache_delete_prepare(struct tango_cache_instance *instance, - struct future* f, const char *objkey, const char *minio_addr, const char *bucket); - -enum OBJECT_LOCATION tango_cache_object_locate(struct tango_cache_instance *instance, size_t object_size); -void tango_cache_get_object_path(struct tango_cache_ctx *ctx, char *path/*OUT*/, size_t pathsize); -struct tango_cache_ctx *tango_cache_update_once_prepare(struct tango_cache_instance *instance, - struct future* f, struct tango_cache_meta_put *meta, size_t object_size, char *path, size_t pathsize); - -#endif - diff --git a/cache/src/tango_cache_pending.cpp b/cache/src/tango_cache_pending.cpp deleted file mode 100644 index 8d803d9..0000000 --- a/cache/src/tango_cache_pending.cpp +++ /dev/null @@ -1,306 +0,0 @@ -#include "tango_cache_pending.h" -#include <tfe_utils.h> -#include <assert.h> -#include <string.h> -#include <stdlib.h> -#include <stdio.h> -#include <stdbool.h> - - -time_t get_time_value(const char* field_value, const char* field_type) -{ - time_t time; - char* time_value = NULL; - field_value += strlen(field_type); - field_value++; - int len = strlen(field_value); - time_value = ALLOC(char, len+1); - int index = 0; - while (field_value[index] != ',' && field_value[index] != '\r' && index < len) - { - time_value[index] = field_value[index]; - index++; - } - time_value[index] = '\0'; - time = (time_t)atol(time_value); - free(time_value); - return time; -} - - -void get_request_freshness(const char *value, struct request_freshness* restrict) -{ - const char* field_value = NULL; - field_value = strstr(value, "min-fresh"); - if (field_value != NULL) - { - restrict->min_fresh = get_time_value(field_value, "min-fresh");; - } - - field_value = strstr(value, "max-age"); - if (field_value != NULL) - { - restrict->max_age = get_time_value(field_value, "max-age");; - } -} - - -enum cache_pending_action request_cache_control(const char* value, struct request_freshness* restrict) -{ - if (strstr(value, "no-cache") != NULL) - { - return REVALIDATE; - } - if (strstr(value, "no-store") != NULL) - { - return FORBIDDEN; - } - get_request_freshness(value, restrict); - return ALLOWED; -} - - -bool cache_verify(const struct tfe_http_half *request) -{ - if( !tfe_http_std_field_read(request,TFE_HTTP_IF_MATCH) || - !tfe_http_std_field_read(request,TFE_HTTP_IF_NONE_MATCH) || - !tfe_http_std_field_read(request,TFE_HTTP_IF_MODIFIED_SINCE) || - !tfe_http_std_field_read(request,TFE_HTTP_IF_UNMODIFIED_SINCE) - ) - { - return true; - } - return false; -} - - -const char* get_head_value(const struct tfe_http_field *http_fields, size_t n_fields, enum tfe_http_std_field head_key) -{ - size_t i = 0; - for (i = 0; i < n_fields; i++) - { - if (http_fields[i].http_field == head_key) - { - return http_fields[i].value; - } - } - return NULL; -} - - -enum cache_pending_action get_pragma_action(const char * value) -{ - const char *pragma_value = "no-cache"; - if (strcasecmp(value, pragma_value) == 0) - { - return REVALIDATE; - } - return UNDEFINED; -} - - -enum cache_pending_action tfe_cache_get_pending(const struct tfe_http_half *request, struct request_freshness* restrict) -{ - enum cache_pending_action res = UNDEFINED; - const char *value = NULL; - memset(restrict,0,sizeof(struct request_freshness)); - if(request->req_spec.method!=TFE_HTTP_METHOD_GET) - { - return FORBIDDEN; - } - if(NULL!=tfe_http_std_field_read(request, TFE_HTTP_CONT_RANGE) || - NULL!=tfe_http_std_field_read(request, TFE_HTTP_AUTHORIZATION)|| - NULL!=tfe_http_nonstd_field_read(request, "WWW-Authenticate")) - { - return FORBIDDEN; - } - value = tfe_http_std_field_read(request, TFE_HTTP_PRAGMA); - if (value != NULL) - { - res = get_pragma_action(value); - } - else - { - value = tfe_http_std_field_read(request, TFE_HTTP_CACHE_CONTROL); - if (value != NULL) - { - res = request_cache_control(value, restrict); - } - else - { - if (cache_verify(request)) - { - res = REVALIDATE; - } - } - } - return res; -} - - - -time_t read_GMT_time(const char* gmt_string) -{ - time_t expire_rel_time; - struct tm expire_gmt_time; - memset(&expire_gmt_time, 0, sizeof(expire_gmt_time)); - strptime(gmt_string, "%a, %d %b %Y %H:%M:%S GMT", &expire_gmt_time); - expire_rel_time = mktime(&expire_gmt_time); - return expire_rel_time; -} - - -bool is_standard_gmt_format(const char* value) -{ - int str_len = strlen(value); - if(0==strcasecmp(value+str_len-3,"GMT")) - { - return true; - } - else - { - return false; - } -} - -time_t get_response_s_maxage(const char* cache_ctl) -{ - const char* s_maxage = NULL; - s_maxage = strstr(cache_ctl, "s-maxage"); - if (s_maxage != NULL) - { - return get_time_value(s_maxage, "s-maxage"); - } - else - { - return 0; - } -} - - -time_t get_response_maxage(const char* cache_ctl) -{ - const char* max_age = NULL; - max_age = strstr(cache_ctl, "max-age"); - if (max_age != NULL) - { - return get_time_value(max_age, "max-age"); - } - else - { - return 0; - } -} - - -void get_response_freshness(const struct tfe_http_half *response, struct response_freshness* freshness) -{ - time_t expire_rel_time = 0; - time_t cur_rel_time = 0; - struct tm cur_gmt_time; - const char* field_value = NULL; - field_value = tfe_http_std_field_read(response, TFE_HTTP_CACHE_CONTROL); - if (field_value != NULL) - { - freshness->timeout = get_response_s_maxage(field_value); - if (freshness->timeout == 0) - { - freshness->timeout = get_response_maxage(field_value); - } - } - else - { - field_value = tfe_http_std_field_read(response, TFE_HTTP_EXPIRES); - if (field_value != NULL && is_standard_gmt_format(field_value)) - { - expire_rel_time = read_GMT_time(field_value); - const time_t cur_ct_time = time(NULL); - if (gmtime_r(&cur_ct_time, &cur_gmt_time) == NULL) - { - assert(0); - } - cur_rel_time = mktime(&cur_gmt_time); - freshness->timeout = expire_rel_time - cur_rel_time; - } - } - field_value = tfe_http_std_field_read(response, TFE_HTTP_DATE); - if (field_value != NULL) - { - if(is_standard_gmt_format(field_value)) - { - freshness->date = read_GMT_time(field_value);; - } - } - field_value = tfe_http_std_field_read(response, TFE_HTTP_LAST_MODIFIED); - if (field_value != NULL && is_standard_gmt_format(field_value)) - { - freshness->last_modified = read_GMT_time(field_value);; - } -} - - -enum cache_pending_action response_cache_control(const char* value) -{ - const char *forbidden_vaule[] = {"no-store", "private"}; - const char *verify_vaule[] = { "no-cache", "must-revalidate","proxy-revalidate" }; - int i = 0; - for (i = 0; i < 2; i++) - { - if (strstr(value, forbidden_vaule[i]) != NULL) - { - return FORBIDDEN; - } - } - for (i = 0; i < 3; i++) - { - if (strstr(value, verify_vaule[i]) != NULL) - { - return REVALIDATE; - } - } - return ALLOWED; -} - - -enum cache_pending_action tfe_cache_put_pending(const struct tfe_http_half *response, struct response_freshness* freshness) -{ - enum cache_pending_action res = UNDEFINED; - const char *value = NULL; - memset(freshness,0,sizeof(struct response_freshness)); - if(response->resp_spec.resp_code!=TFE_HTTP_STATUS_OK - || NULL!=tfe_http_std_field_read(response, TFE_HTTP_CONT_RANGE) //NOT upload response with content-range - || NULL==response->resp_spec.content_length - || NULL!=tfe_http_std_field_read(response, TFE_HTTP_AUTHORIZATION) - || NULL!=tfe_http_nonstd_field_read(response, "WWW-Authenticate") - || NULL!=tfe_http_std_field_read(response, TFE_HTTP_SET_COOKIE)) - { - return FORBIDDEN; - } - - value = tfe_http_std_field_read(response, TFE_HTTP_PRAGMA); - if (value != NULL) - { - res = get_pragma_action(value); - } - else - { - value = tfe_http_std_field_read(response, TFE_HTTP_CACHE_CONTROL); - if (value != NULL) - { - res = response_cache_control(value); - } - else - { - value = tfe_http_std_field_read(response, TFE_HTTP_EXPIRES); - if (value != NULL && 0!= read_GMT_time(value)) - { - res = ALLOWED; - } - } - } - if (res == ALLOWED) - { - get_response_freshness(response, freshness); - } - return res; -} diff --git a/cache/src/tango_cache_redis.cpp b/cache/src/tango_cache_redis.cpp deleted file mode 100644 index 54fd8d0..0000000 --- a/cache/src/tango_cache_redis.cpp +++ /dev/null @@ -1,422 +0,0 @@ -#include <sys/types.h> -#include <unistd.h> -#include <stdio.h> -#include <stdlib.h> -#include <assert.h> -#include <errno.h> -#include <sys/time.h> -#include <time.h> -#include <string.h> - -#include <hiredis-vip/hircluster.h> -#include <hiredis-vip/async.h> -#include <hiredis-vip/adapters/libevent.h> -#include <cjson/cJSON.h> - -#include "tango_cache_transfer.h" -#include "tango_cache_tools.h" -#include "tango_cache_redis.h" - -#define PARSE_JSON_RET_ERROR -1 -#define PARSE_JSON_RET_TIMEOUT 0 -#define PARSE_JSON_RET_SUCC 1 - -#define CACHE_REDIS_CONNECT_IDLE 0 -#define CACHE_REDIS_CONNECTING 1 -#define CACHE_REDIS_CONNECTED 2 -#define CACHE_REDIS_DISCONNECTED 3 - -static void redis_asyn_disconnect_cb(const struct redisAsyncContext *ac, int status) -{ - struct tango_cache_instance *instance = (struct tango_cache_instance *)redisAsyncGetConnectionData(ac); - - if(status == REDIS_OK) - { - MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Redis disconnect %s:%d success.", - ac->c.tcp.host, ac->c.tcp.port); - } - else - { - MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "Redis disconnect %s:%d failed: %s.", - ac->c.tcp.host, ac->c.tcp.port, ac->errstr); - } - instance->redis_connecting = CACHE_REDIS_DISCONNECTED; -} - -static void redis_asyn_connect_cb(const struct redisAsyncContext *ac, int status) -{ - struct tango_cache_instance *instance = (struct tango_cache_instance *)redisAsyncGetConnectionData(ac); - - if(status == REDIS_OK) - { - MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "RedisCluster connect %s:%d success.", - ac->c.tcp.host, ac->c.tcp.port); - instance->redis_connecting = CACHE_REDIS_CONNECTED; - } - else - { - instance->redis_connecting = CACHE_REDIS_CONNECT_IDLE; - MESA_HANDLE_RUNTIME_LOGV2(instance->runtime_log, RLOG_LV_FATAL, "RedisCluster connect %s:%d failed: %s.", - ac->c.tcp.host, ac->c.tcp.port, ac->errstr); - } -} - -int redis_asyn_connect_init(struct tango_cache_instance *instance) -{ - instance->redis_ac = redisClusterAsyncConnect(instance->param->redisaddrs, HIRCLUSTER_FLAG_ROUTE_USE_SLOTS); - if(instance->redis_ac == NULL) - { - return -1; - } - instance->redis_connecting = CACHE_REDIS_CONNECTING; - redisClusterLibeventAttach(instance->redis_ac, instance->evbase); - redisClusterAsyncSetConnectionData(instance->redis_ac, instance); - redisClusterAsyncSetConnectCallback(instance->redis_ac, redis_asyn_connect_cb); - redisClusterAsyncSetDisconnectCallback(instance->redis_ac, redis_asyn_disconnect_cb); - return 0; -} - -static int parse_object_meta_json(struct tango_cache_ctx *ctx, const char *jcontent) -{ - cJSON *root, *ptarget; - int ret = PARSE_JSON_RET_ERROR; - char usertag[2048]; - size_t datalen; - - if(NULL == (root=cJSON_Parse(jcontent))) - { - goto out_json; - } - if(NULL == (ptarget=cJSON_GetObjectItem(root, "Content-Length"))) - { - goto out_json; - } - ctx->get.result.tlength = ptarget->valuedouble; - if(NULL==(ptarget=cJSON_GetObjectItem(root, "X-Amz-Meta-Lm"))) - { - goto out_json; - } - ctx->get.last_modify = ptarget->valuedouble; - if(NULL==(ptarget=cJSON_GetObjectItem(root, "Expires"))) - { - goto out_json; - } - ctx->get.expires = ptarget->valuedouble; - ctx->get.need_hdrs = RESPONSE_HDR_ALL; - if(!check_expires_fresh_header(ctx)) - { - ret = PARSE_JSON_RET_TIMEOUT; - goto out_json; - } - - if(NULL!=(ptarget=cJSON_GetObjectItem(root, "Headers"))) - { - easy_string_savedata(&ctx->response, ptarget->valuestring, strlen(ptarget->valuestring)); - } - if(NULL!=(ptarget=cJSON_GetObjectItem(root, "X-Amz-Meta-User"))) - { - if((datalen = Base64_DecodeBlock((unsigned char*)ptarget->valuestring, strlen(ptarget->valuestring), (unsigned char*)usertag, 2048))>0) - { - easy_string_savedata(&ctx->get.response_tag, usertag, datalen); - } - } - cJSON_Delete(root); - return PARSE_JSON_RET_SUCC; - -out_json: - cJSON_Delete(root); - return ret; -} - -static void redis_hget_command_cb(struct redisClusterAsyncContext *ac, void *vreply, void *privdata) -{ - redisReply *reply = (redisReply *)vreply; - struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)privdata; - int ret; - - ctx->instance->statistic.session_redis -= 1; - if(reply == NULL || reply->type!=REDIS_REPLY_ARRAY) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_EXEC); - if(reply!=NULL && reply->type==REDIS_REPLY_ERROR) - { - promise_failed(ctx->promise, FUTURE_ERROR_CANCEL, reply->str); - } - else - { - promise_failed(ctx->promise, FUTURE_ERROR_CANCEL, tango_cache_get_errstring(ctx)); - } - tango_cache_ctx_destroy(ctx); - return; - } - else if(reply->element[0]->type == REDIS_REPLY_NIL) - { - tango_cache_set_fail_state(ctx, CACHE_CACHE_MISS); - ctx->get.result.type = RESULT_TYPE_MISS; - promise_success(ctx->promise, &ctx->get.result); - promise_finish(ctx->promise); - tango_cache_ctx_destroy(ctx); - return; - } - - switch(ctx->get.state) - { - case GET_STATE_REDIS_META: - ctx->get.result.location = (strcmp(reply->element[1]->str, "redis"))?OBJECT_IN_HOS:OBJECT_IN_REDIS; - break; - case GET_STATE_REDIS_ALL: - ctx->get.result.location = OBJECT_IN_REDIS; - break; - - case GET_STATE_REDIS_TRY: - ctx->get.result.location = (strcmp(reply->element[1]->str, "redis"))?OBJECT_IN_HOS:OBJECT_IN_REDIS; - if(ctx->get.result.location == OBJECT_IN_HOS) - { - ctx->get.redis_redirect_minio_cb(ctx); - return; - } - ctx->locate = OBJECT_IN_REDIS; - break; - default: assert(0);break; - } - - ret = parse_object_meta_json(ctx, reply->element[0]->str); - switch(ret) - { - case PARSE_JSON_RET_ERROR: - tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_JSON); - promise_failed(ctx->promise, FUTURE_ERROR_CANCEL, tango_cache_get_errstring(ctx)); - tango_cache_ctx_destroy(ctx); - break; - case PARSE_JSON_RET_TIMEOUT: - if(ctx->get.state == GET_STATE_DELETE && ctx->get.result.location==OBJECT_IN_HOS) - { - ctx->get.state = GET_STATE_END; - cache_delete_minio_object(ctx); - } - else - { - tango_cache_ctx_destroy(ctx); - } - break; - case PARSE_JSON_RET_SUCC: - fetch_header_over_biz(ctx); - if(ctx->get.state != GET_STATE_REDIS_META) - { - ctx->get.result.data_frag = reply->element[2]->str; - ctx->get.result.size = reply->element[2]->len; - ctx->get.result.type = RESULT_TYPE_BODY; - promise_success(ctx->promise, &ctx->get.result); - } - ctx->get.result.type = RESULT_TYPE_END; - promise_success(ctx->promise, &ctx->get.result); - promise_finish(ctx->promise); - tango_cache_ctx_destroy(ctx); - break; - default: assert(0);break; - } -} - -int tango_cache_head_redis(struct tango_cache_ctx *ctx) -{ - int ret = -1; - - ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hget_command_cb, ctx, - "HMGET %s OBJECT_META OBJECT_LOCATION", ctx->object_key); - if(ret != REDIS_OK) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT); - tango_cache_ctx_destroy(ctx); - } - else - { - ctx->instance->statistic.session_redis += 1; - ctx->get.state = GET_STATE_REDIS_META; - } - return ret; -} - -int tango_cache_fetch_redis(struct tango_cache_ctx *ctx) -{ - int ret = -1; - - ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hget_command_cb, ctx, - "HMGET %s OBJECT_META OBJECT_LOCATION OBJECT_BODY", ctx->object_key); - if(ret != REDIS_OK) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT); - tango_cache_ctx_destroy(ctx); - } - else - { - ctx->instance->statistic.session_redis += 1; - ctx->get.state = GET_STATE_REDIS_ALL; - } - return ret; -} - -int tango_cache_try_fetch_redis(struct tango_cache_ctx *ctx) -{ - int ret = -1; - - ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hget_command_cb, ctx, - "HMGET %s OBJECT_META OBJECT_LOCATION OBJECT_BODY", ctx->object_key); - if(ret != REDIS_OK) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT); - tango_cache_ctx_destroy(ctx); - } - else - { - ctx->instance->statistic.session_redis += 1; - ctx->get.state = GET_STATE_REDIS_TRY; - } - return ret; -} - -static void redis_hset_command_cb(struct redisClusterAsyncContext *ac, void *vreply, void *privdata) -{ - struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)privdata; - redisReply *reply = (redisReply *)vreply; - int ret; - - ctx->instance->statistic.session_redis -= 1; - if(reply == NULL || reply->type==REDIS_REPLY_ERROR || ac->err) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_EXEC); - } - if(ctx->fail_state) - { - if(ctx->put.state==PUT_STATE_REDIS_META || ctx->put.state==PUT_STATE_REDIS_SETEX) //����һ��������ص� - { - ctx->put.state = PUT_STATE_END; - return; - } - tango_cache_ctx_destroy(ctx, true); - return; - } - - switch(ctx->put.state) - { - case PUT_STATE_REDIS_META: - ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hset_command_cb, ctx, - "EXPIRE %s %u", ctx->object_key, ctx->put.object_ttl); - if(ret!=REDIS_OK) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT); - ctx->put.state = PUT_STATE_END; - } - else - { - ctx->instance->statistic.session_redis += 1; - ctx->put.state = PUT_STATE_REDIS_SETEX; - } - break; - case PUT_STATE_REDIS_EXPIRE: - ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hset_command_cb, ctx, - "EXPIRE %s %u", ctx->object_key, ctx->put.object_ttl); - if(ret != REDIS_OK) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT); - tango_cache_ctx_destroy(ctx, true); - } - else - { - ctx->instance->statistic.session_redis += 1; - ctx->put.state = PUT_STATE_END; - } - break; - case PUT_STATE_REDIS_SETEX: - ctx->put.state = PUT_STATE_END; //����һ��EXPIRE��������� - break; - case PUT_STATE_END: - tango_cache_ctx_destroy(ctx, true); - break; - default: assert(0);break; - } -} - -int redis_put_minio_object_meta(struct tango_cache_ctx *ctx, bool callback) -{ - int ret_mset, ret_set; - char *meta; - - meta = cJSON_PrintUnformatted(ctx->put.object_meta); - - ret_mset = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hset_command_cb, ctx, - "HMSET %s OBJECT_LOCATION minio OBJECT_META %s MINIO_ADDR %s", ctx->object_key, meta, ctx->hostaddr); - ret_set = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hset_command_cb, ctx, - "SET http://%s/%s 1 EX %u", ctx->hostaddr, ctx->object_key, ctx->put.object_ttl); - if(ret_mset==REDIS_OK && ret_set==REDIS_OK) - { - ctx->instance->statistic.session_redis += 2; - ctx->put.state = PUT_STATE_REDIS_META; - } - else - { - tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT); - if(ret_mset==REDIS_OK) - { - ctx->instance->statistic.session_redis += 1; - ctx->put.state = PUT_STATE_REDIS_EXPIRE; //��ʱ��PUT����object��һ�� - } - else if(ret_set==REDIS_OK) - { - ctx->instance->statistic.session_redis += 1; - ctx->put.state = PUT_STATE_END; - } - else - { - tango_cache_ctx_destroy(ctx, callback); - free(meta); - return -1; - } - } - free(meta); - return 0; -} - -int redis_put_complete_part_data(struct tango_cache_ctx *ctx, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, bool callback) -{ - int ret; - char *meta; - - ctx->instance->statistic.memory_used -= size; - meta = cJSON_PrintUnformatted(ctx->put.object_meta); - ret = redisClusterAsyncCommand(ctx->instance->redis_ac, redis_hset_command_cb, ctx, - "HMSET %s OBJECT_LOCATION redis OBJECT_META %s OBJECT_BODY %b", ctx->object_key, meta, data, size); - if(ret != REDIS_OK) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_REDIS_CONNECT); - tango_cache_ctx_destroy(ctx, callback); - } - else - { - ctx->instance->statistic.session_redis += 1; - ctx->put.state = PUT_STATE_REDIS_EXPIRE; - } - if(way == PUT_MEM_FREE) - { - free((void *)data); - } - free(meta); - return ret; -} - -int redis_put_complete_part_evbuf(struct tango_cache_ctx *ctx, size_t object_size, bool callback) -{ - char *data; - size_t size; - - data = (char *)malloc(object_size); - size = evbuffer_remove(ctx->put.evbuf, data, object_size); - if(size != object_size) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_EVBUFFER); - tango_cache_ctx_destroy(ctx, callback); - free(data); - return CACHE_ERR_EVBUFFER; - } - return redis_put_complete_part_data(ctx, PUT_MEM_FREE, data, object_size, callback); -} - diff --git a/cache/src/tango_cache_redis.h b/cache/src/tango_cache_redis.h deleted file mode 100644 index 74e11e4..0000000 --- a/cache/src/tango_cache_redis.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef __TANGO_CACHE_REDIS_H__ -#define __TANGO_CACHE_REDIS_H__ - -#include <event2/event.h> -#include <event.h> - -#include "tango_cache_client_in.h" - -int tango_cache_head_redis(struct tango_cache_ctx *ctx); -int redis_asyn_connect_init(struct tango_cache_instance *instance); - - -int tango_cache_fetch_redis(struct tango_cache_ctx *ctx); -int tango_cache_try_fetch_redis(struct tango_cache_ctx *ctx); - -int redis_put_minio_object_meta(struct tango_cache_ctx *ctx, bool callback); -int redis_put_complete_part_data(struct tango_cache_ctx *ctx, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, bool callback); -int redis_put_complete_part_evbuf(struct tango_cache_ctx *ctx, size_t object_size, bool callback); - -#endif - diff --git a/cache/src/tango_cache_tools.cpp b/cache/src/tango_cache_tools.cpp deleted file mode 100644 index e22a8b9..0000000 --- a/cache/src/tango_cache_tools.cpp +++ /dev/null @@ -1,258 +0,0 @@ -#include <sys/time.h> -#include <time.h> -#include <string.h> -#include <ctype.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> -#include <stdio.h> -#include <stdlib.h> -#include <errno.h> - -#include "tango_cache_tools.h" - -static const char data_bin2ascii[65] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - -#define conv_bin2ascii(a) (data_bin2ascii[(a)&0x3f]) - -#define B64_EOLN 0xF0 -#define B64_CR 0xF1 -#define B64_EOF 0xF2 -#define B64_WS 0xE0 -#define B64_ERROR 0xFF -#define B64_NOT_BASE64(a) (((a)|0x13) == 0xF3) -#define B64_BASE64(a) !B64_NOT_BASE64(a) - -static const unsigned char data_ascii2bin[128] = { - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xE0, 0xF0, 0xFF, 0xFF, 0xF1, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xE0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0x3E, 0xFF, 0xF2, 0xFF, 0x3F, - 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, - 0x3C, 0x3D, 0xFF, 0xFF, 0xFF, 0x00, 0xFF, 0xFF, - 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, - 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, - 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, - 0x17, 0x18, 0x19, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, - 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, - 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30, - 0x31, 0x32, 0x33, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, -}; - -//�мDz�������˵���: conv_ascii2bin(x++); ��Ϊ��++���� -#define conv_ascii2bin(aa) (((aa) & 0x80)?(0xFF):data_ascii2bin[(aa)]) - -/********************************************************************* -�������ƣ�Base64_EncodeBlock -���ܼ�飺��һ����BASE64�������б��� -���������in����������ַ��� - inl��in�ij��� -���������out�������洢�Ļ������� -����ֵ�������ij��� -*********************************************************************/ -int Base64_EncodeBlock(const unsigned char *in, int inl, unsigned char *out) -{ - int i, ret = 0; - unsigned long l; - - for (i = inl; i > 0; i -= 3) { - if (i >= 3) { - l = (((unsigned long)in[0]) << 16L) | - (((unsigned long)in[1]) << 8L) | in[2]; - *(out++) = conv_bin2ascii(l >> 18L); - *(out++) = conv_bin2ascii(l >> 12L); - *(out++) = conv_bin2ascii(l >> 6L); - *(out++) = conv_bin2ascii(l); - } else { - l = ((unsigned long)in[0]) << 16L; - if (i == 2) - l |= ((unsigned long)in[1] << 8L); - - *(out++) = conv_bin2ascii(l >> 18L); - *(out++) = conv_bin2ascii(l >> 12L); - *(out++) = (i == 1) ? '=' : conv_bin2ascii(l >> 6L); - *(out++) = '='; - } - ret += 4; - in += 3; - } - - *out = '\0'; - return (ret); -} - -/********************************************************************* -�������ƣ�Base64_DecodeBlock -���ܼ�飺��һ����BASE64�������н������Զ�������β��BASE64�����ַ� -���������in����������ַ��� - inl��in�ij��� -���������out�������洢�Ļ������� - ���뱣֤���㹻�Ŀռ䣬һ��ﵽ@inl��С���ɣ� -����ֵ��<0��ʧ�ܣ�>=0�������ij��� -*********************************************************************/ -int Base64_DecodeBlock(const unsigned char *in, int inl, unsigned char *out, int outsize) -{ - int i, ret = 0; - unsigned char a, b, c, d; - unsigned long l; - - /* ignore not-base64-encoded charactor. */ - while ((conv_ascii2bin(*in) == B64_WS) && (inl > 0)) - { - in++; - inl--; - } - while ((inl > 3) && (B64_NOT_BASE64(conv_ascii2bin(in[inl - 1])))) - inl--; - - if (inl % 4 != 0) - return -1; - - if(outsize < (inl*3)/4) - return -2; - - for (i = 0; i < inl; i += 4) - { - a = conv_ascii2bin(*(in)); - b = conv_ascii2bin(*(in+1)); - c = conv_ascii2bin(*(in+2)); - d = conv_ascii2bin(*(in+3)); - if ((a & 0x80) || (b & 0x80) || (c & 0x80) || (d & 0x80)) - return (-1); - l = ((((unsigned long)a) << 18L) | (((unsigned long)b) << 12L) | - (((unsigned long)c) << 6L) | (((unsigned long)d))); - *(out++) = (unsigned char)(l >> 16L) & 0xff; - *(out++) = (unsigned char)(l >> 8L) & 0xff; - *(out++) = (unsigned char)(l) & 0xff; - ret += 3; - in+=4; - } - - for(i = inl; i > 0; i -= 4) - { - if(*(in-3) == '=') - { - in -= 4; - ret -= 3; - continue; - } - - while(*(--in) == '=') - ret -= 1; - - break; - } - - return ret; -} - - -//��֪������������ͬ������£��Ƚ��������Ƿ���� -int strcmp_one_word_mesa_equal_len(const char *s1_lowercase, const char *s1_uppercase, const char *s2, size_t len) -{ - unsigned char *s1,*s12; - - if (s2[len-1]-'a'>=0) - { - s1 = (unsigned char *)s1_lowercase; - s12= (unsigned char *)s1_uppercase; - } - else - { - s1 = (unsigned char *)s1_uppercase; - s12= (unsigned char *)s1_lowercase; - } - - do { - if (*s1 == *s2 || *s12 == *s2) - { - ++s1; - ++s12; - ++s2; - continue; - } - return 0; - } while (--len); - - return 1; -} - -int mkdir_according_path(const char * path) -{ - char buffer[256]; - const char *ps=path, *pc; - - if(*ps == '/') - ps += 1; - - while((pc = strchr(ps, '/')) != NULL) - { - while(*(pc+1) == '/') - pc++; - - memcpy(buffer, path, pc - path); - buffer[pc-path] = '\0'; - - if(access(buffer, F_OK)) - { - if(mkdir(buffer, 0777) && errno!=EEXIST) - { - return -1; - } - } - - ps = pc + 1; - } - if(access(path, F_OK)) - { - if(mkdir(path, 0777)) - { - return -1; - } - } - return 0; -} - -//��ʱ���ַ���ת��Ϊʱ��� -time_t expires_hdr2timestamp(const char *expires_val, int len) -{ - struct tm tm; - - while(len > 0 && (*expires_val==' '||*expires_val=='\t'||*expires_val=='\r'||*expires_val=='\n')) - { - expires_val++; - len--; - } - if(len == 0) - { - return 0; - } - - memset(&tm, 0, sizeof(struct tm)); - if(strptime(expires_val, "%a, %d %b %Y %T", &tm) == NULL) - { - return 0; - } - - return mktime(&tm); -} - -//������ʱ���ת��ΪGMTʱ���ַ��� -size_t expires_timestamp2hdr_str(time_t seconds, char *buffer, size_t size) -{ - struct tm save; - return strftime(buffer, size, "Expires: %a, %d %b %Y %T GMT", gmtime_r(&seconds, &save)); -} - -//������ʱ���ת��ΪGMTʱ��� -time_t get_gmtime_timestamp(time_t seconds) -{ - struct tm *tm, save; - - tm = gmtime_r(&seconds, &save); - return mktime(tm); -} - diff --git a/cache/src/tango_cache_tools.h b/cache/src/tango_cache_tools.h deleted file mode 100644 index e0ab730..0000000 --- a/cache/src/tango_cache_tools.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef __TANGO_CACHE_TOOLS_H__ -#define __TANGO_CACHE_TOOLS_H__ - -#include <MESA/MESA_handle_logger.h> - -#define MESA_HANDLE_RUNTIME_LOGV2(handle, lv, fmt, args...) \ - MESA_handle_runtime_log((handle), (lv), "TANGO_CACHE", "%s:%d, " fmt, __FILE__, __LINE__, ##args) - -#ifdef CACHE_DEBUG_SWITCH -#define DBG_CACHE(fmt, args...) do{printf("%s():%d, " fmt, __FUNCTION__, __LINE__, ##args);}while(0) -#else -#define DBG_CACHE(msg...) -#endif - -int Base64_EncodeBlock(const unsigned char *in, int inl, unsigned char *out); -int Base64_DecodeBlock(const unsigned char *in, int inl, unsigned char *out, int outsize); - -int strcmp_one_word_mesa_equal_len(const char *s1_lowercase, const char *s1_uppercase, const char *s2, size_t len); -int mkdir_according_path(const char * path); - -time_t get_gmtime_timestamp(time_t seconds); -time_t expires_hdr2timestamp(const char *expires_val, int len); -size_t expires_timestamp2hdr_str(time_t seconds, char *buffer, size_t size); - -#endif - diff --git a/cache/src/tango_cache_transfer.cpp b/cache/src/tango_cache_transfer.cpp deleted file mode 100644 index e8b4cfc..0000000 --- a/cache/src/tango_cache_transfer.cpp +++ /dev/null @@ -1,986 +0,0 @@ -#include <sys/types.h> -#include <unistd.h> -#include <stdio.h> -#include <stdlib.h> -#include <assert.h> -#include <errno.h> -#include <sys/time.h> -#include <time.h> -#include <string.h> -#include <curl/curl.h> - -#include "tango_cache_transfer.h" -#include "tango_cache_xml.h" -#include "tango_cache_tools.h" -#include "tango_cache_redis.h" - -static inline void curl_set_common_options(CURL *curl, long transfer_timeout, char *errorbuf) -{ - curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errorbuf); - curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, 500L); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, transfer_timeout); //���Է��ֶ�������ij���ӽ��տ�ס����� - //ctx->error="Operation too slow. Less than 1024 bytes/sec transferred the last 3 seconds" - curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 5L); - curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 100L); - curl_easy_setopt(curl, CURLOPT_USERAGENT, "aws-sdk-cpp/1.5.24 Linux/3.10.0-327.el7.x86_64 x86_64 pangu_cache"); -} - -//response body�̻ܶ���ʱ -size_t curl_response_any_cb(void *ptr, size_t size, size_t count, void *userp) -{ - return size*count; -} - -static size_t curl_put_multipart_header_cb(void *ptr, size_t size, size_t count, void *userp) -{ - struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp; - size_t totallen = size*count; - char *start = (char *)ptr, *end = start + totallen; - struct multipart_etag_list *etag; - - if(!strncmp(start, "Etag:", totallen>5?5:totallen)) - { - start += 5; end -= 1; totallen -= 5; - while(totallen>0 && (*start==' ')) {start++; totallen--;} - while(totallen>0 && (*end=='\r'||*end=='\n')) {end--; totallen--;} - if(totallen > 0) - { - etag = (struct multipart_etag_list *)malloc(sizeof(struct multipart_etag_list)); - totallen = end - start + 1; - etag->etag = (char *)malloc(totallen + 1); - etag->part_number = ctx->put.part_index; - memcpy(etag->etag, start, totallen); - *(etag->etag + totallen) = '\0'; - TAILQ_INSERT_TAIL(&ctx->put.etag_head, etag, node); - } - } - - return size*count; -} - -static size_t curl_put_once_send_cb(void *ptr, size_t size, size_t count, void *userp) -{ - size_t len; - struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp; - - if(size==0 || count==0 || ctx->put.once_request.len>=ctx->put.once_request.size) - { - return 0; //��һ������ - } - - len = ctx->put.once_request.size - ctx->put.once_request.len; //ʣ����ϴ��ij��� - if(len > size * count) - { - len = size * count; - } - - memcpy(ptr, ctx->put.once_request.buff + ctx->put.once_request.len, len); - ctx->put.once_request.len += len; - - if(ctx->put.once_request.len >= ctx->put.once_request.size) - { - ctx->instance->statistic.memory_used -= ctx->put.once_request.size; //δʹ��cache buffer���Լ������ڴ����� - easy_string_destroy(&ctx->put.once_request); - } - return len; -} - -static size_t curl_put_multipart_send_cb(void *ptr, size_t size, size_t count, void *userp) -{ - size_t len, space=size*count, send_len; - struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp; - - if(size==0 || count==0 || ctx->put.upload_offset>=ctx->put.upload_length) - { - return 0; - } - - len = ctx->put.upload_length - ctx->put.upload_offset; - if(len > space) - { - len = space; - } - send_len = evbuffer_remove(ctx->put.evbuf, ptr, len); - assert(send_len>0); - ctx->put.upload_offset += send_len; - ctx->instance->statistic.memory_used -= send_len; - - return send_len; -} - -//return value: <0:fail; =0: not exec; >0: OK -static int http_put_bodypart_request_evbuf(struct tango_cache_ctx *ctx, bool full) -{ - UNUSED CURLMcode rc; - char minio_url[256]={0}, buffer[256]={0}; - - if(NULL == (ctx->curl=curl_easy_init())) - { - return -1; - } - - ctx->put.upload_offset = 0; - if(full) - { - snprintf(minio_url, 256, "http://%s/%s", ctx->hostaddr, ctx->object_key); - } - else - { - snprintf(minio_url, 256, "http://%s/%s?partNumber=%d&uploadId=%s", ctx->hostaddr, ctx->object_key, ++ctx->put.part_index, ctx->put.uploadID); - curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_put_multipart_header_cb); - curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx); - } - curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); - curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb); - curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); - //token�ֶΣ�����hos�洢��֤ - sprintf(buffer, "token: %s", ctx->instance->param->cache_token); - ctx->headers = curl_slist_append(ctx->headers, buffer); - curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); - curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L); - curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->put.upload_length); - curl_easy_setopt(ctx->curl, CURLOPT_READFUNCTION, curl_put_multipart_send_cb); - curl_easy_setopt(ctx->curl, CURLOPT_READDATA, ctx); - curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error); - - rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl); - assert(rc==CURLM_OK); - DBG_CACHE("state: %d, length: %lu, key: %s\n", ctx->put.state, ctx->put.upload_length, ctx->object_key); - return 1; -} - -static size_t curl_response_body_save_cb(void *ptr, size_t size, size_t count, void *userp) -{ - struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp; - struct easy_string *estr = &ctx->response; - CURLcode code; - - if(ctx->fail_state) - { - return size*count; - } - - if(ctx->res_code == 0) - { - code = curl_easy_getinfo(ctx->curl, CURLINFO_RESPONSE_CODE, &ctx->res_code); - if(code != CURLE_OK || ctx->res_code!=200L) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_CURL); - return size*count; - } - } - - easy_string_savedata(estr, (const char*)ptr, size*count); - return size*count; -} - -int curl_get_minio_uploadID(struct tango_cache_ctx *ctx) -{ - UNUSED CURLMcode rc; - char minio_url[256]={0}, buffer[256]; - - if(NULL == (ctx->curl=curl_easy_init())) - { - return -1; - } - - snprintf(minio_url, 256, "http://%s/%s?uploads", ctx->hostaddr, ctx->object_key); - curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L); - curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, 0); //Ĭ��ʹ�ûص���������fread�����Է��ֹر�Expectʱ�ᵼ�¿���curl_multi_socket_action - curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); - - curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_body_save_cb); - curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); - sprintf(buffer, "token: %s", ctx->instance->param->cache_token); - ctx->headers = curl_slist_append(ctx->headers, buffer); - curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); - curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error); - - rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl); - assert(rc==CURLM_OK); - DBG_CACHE("state: %d, key: %s\n", ctx->put.state, ctx->object_key); - return 1; -} - -int cache_delete_minio_object(struct tango_cache_ctx *ctx, bool call_back) -{ - UNUSED CURLMcode rc; - char minio_url[256], buffer[256]; - - ctx->instance->statistic.del_recv_num += 1; - if(NULL == (ctx->curl=curl_easy_init())) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_CURL); - tango_cache_ctx_destroy(ctx, call_back); //�ս��� - return -1; - } - - snprintf(minio_url, 256, "http://%s/%s", ctx->hostaddr, ctx->object_key); - curl_easy_setopt(ctx->curl, CURLOPT_CUSTOMREQUEST, "DELETE"); - curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); - curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb); - curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); - sprintf(buffer, "token: %s", ctx->instance->param->cache_token); - ctx->headers = curl_slist_append(ctx->headers, buffer); - curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); - curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error); - - rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl); - assert(rc==CURLM_OK); - return 1; -} - -//return value: true-�ɹ������¼���false-δ�����¼� -bool cache_cancel_upload_minio(struct tango_cache_ctx *ctx) -{ - UNUSED CURLMcode rc; - char minio_url[256]; - - if(NULL == (ctx->curl=curl_easy_init())) - { - return false; - } - - snprintf(minio_url, 256, "http://%s/%s?uploadId=%s", ctx->hostaddr, ctx->object_key, ctx->put.uploadID); - curl_easy_setopt(ctx->curl, CURLOPT_CUSTOMREQUEST, "DELETE"); - curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); - curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb); - curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); - curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error); - - rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl); - assert(rc==CURLM_OK); - return true; -} - -//return value: true-�ɹ������¼���false-δ�����¼� -bool cache_kick_combine_minio(struct tango_cache_ctx *ctx) -{ - int len=0; - UNUSED CURLMcode rc; - char minio_url[256], buffer[256]; - - if(NULL == (ctx->curl=curl_easy_init())) - { - return false; - } - construct_complete_xml(ctx, &ctx->put.combine_xml, &len); - - snprintf(minio_url, 256, "http://%s/%s?uploadId=%s", ctx->hostaddr, ctx->object_key, ctx->put.uploadID); - curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L); - curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); - curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb); - curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); - - curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, len); //���Content-Length - curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDS, ctx->put.combine_xml); - curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error); - - if(ctx->headers != NULL) - { - curl_slist_free_all(ctx->headers); - ctx->headers = NULL; - } - ctx->headers = curl_slist_append(ctx->headers, "Content-Type: application/xml"); - sprintf(buffer, "token: %s", ctx->instance->param->cache_token); - ctx->headers = curl_slist_append(ctx->headers, buffer); - curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); - - rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl); - assert(rc==CURLM_OK); - DBG_CACHE("state: %d, key: %s\n", ctx->put.state, ctx->object_key); - return true; -} - -//return value: true-�ɹ������¼���false-δ�����¼� -bool cache_kick_upload_minio_multipart(struct tango_cache_ctx *ctx, size_t block_len) -{ - int ret = 1; - - switch(ctx->put.state) - { - case PUT_STATE_START: - if(sessions_exceeds_limit(ctx->instance, OBJECT_IN_HOS)) - { - tango_cache_set_fail_state(ctx, CACHE_OUTOF_SESSION); - return false; - } - ctx->put.state = PUT_STATE_WAIT_START; - ret = curl_get_minio_uploadID(ctx); - break; - - case PUT_STATE_PART: - if(ctx->curl == NULL) - { - ctx->put.upload_length = block_len; - ret = http_put_bodypart_request_evbuf(ctx, false); - } - break; - - default: break;//nothing to do - } - - if(ret <= 0) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_CURL); - return false; - } - return true; -} - -//callbackֱ��ʧ���Ƿ���ûص���������ʽ��Ҫ������һ���Բ���Ҫ -static int http_put_complete_part_evbuf(struct tango_cache_ctx *ctx, bool callback) -{ - int ret=-1; - - ctx->put.state = PUT_STATE_END; - ctx->put.upload_length = evbuffer_get_length(ctx->put.evbuf); - if(ctx->put.upload_length > 0) - { - ret = http_put_bodypart_request_evbuf(ctx, true); - if(ret <= 0) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_CURL); - tango_cache_ctx_destroy(ctx, callback); - } - } - else - { - tango_cache_ctx_destroy(ctx, callback); - } - return ret; -} - -int do_tango_cache_update_end(struct tango_cache_ctx *ctx, bool callback) -{ - DBG_CACHE("state: %d, key: %s, curl %s NULL\n", ctx->put.state, ctx->object_key, (ctx->curl==NULL)?"is":"is not"); - ctx->put.close_state = true;//������״̬�����������رգ��ڲ�״̬����ת�������ٹر� - if(ctx->fail_state) - { - tango_cache_ctx_destroy(ctx, callback); - return -1; - } - - switch(ctx->put.state) - { - case PUT_STATE_START: //��ʱ��ͬ����һ�����ϴ� - if(sessions_exceeds_limit(ctx->instance, ctx->locate)) - { - tango_cache_set_fail_state(ctx, CACHE_OUTOF_SESSION); - tango_cache_ctx_destroy(ctx, callback); - return -1; - } - if(ctx->locate == OBJECT_IN_HOS) - { - return http_put_complete_part_evbuf(ctx, callback); - } - else - { - return redis_put_complete_part_evbuf(ctx, ctx->put.object_size, callback); - } - break; - - case PUT_STATE_PART: - if(ctx->curl == NULL) - { - ctx->put.upload_length = evbuffer_get_length(ctx->put.evbuf); - if(ctx->put.upload_length == 0) - { - if(cache_kick_combine_minio(ctx)) - { - ctx->put.state = PUT_STATE_END; - } - else - { - tango_cache_set_fail_state(ctx, CACHE_ERR_CURL); - tango_cache_ctx_destroy(ctx); - return -1; - } - } - else if(http_put_bodypart_request_evbuf(ctx, false) <= 0) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_CURL); - if(cache_cancel_upload_minio(ctx)) - { - ctx->put.state = PUT_STATE_CANCEL; - } - else - { - tango_cache_ctx_destroy(ctx); - return -1; - } - } - } - break; - - case PUT_STATE_END: assert(0); //�û���������endʱ�����ܴ��ڴ�״̬ - case PUT_STATE_WAIT_START: //��ʱδ��ȡ��uploadId�������������ϴ� - default: break; - } - return 0; -} - -void tango_cache_curl_put_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code) -{ - DBG_CACHE("state: %d, key: %s\n", ctx->put.state, ctx->object_key); - switch(ctx->put.state) - { - case PUT_STATE_WAIT_START: - if(res!=CURLE_OK||res_code!=200L|| ctx->fail_state || !parse_uploadID_xml(ctx->response.buff, ctx->response.len, &ctx->put.uploadID)) - { - easy_string_destroy(&ctx->response); - tango_cache_set_fail_state(ctx, CACHE_ERR_CURL); - if(ctx->put.close_state) - { - tango_cache_ctx_destroy(ctx); - } - } - else - { - easy_string_destroy(&ctx->response); - ctx->put.state = PUT_STATE_PART; - if(ctx->put.close_state) - { - do_tango_cache_update_end(ctx, true); - } - else - { - size_t upload_length = evbuffer_get_length(ctx->put.evbuf); - if(upload_length >= ctx->instance->param->upload_block_size) - { - cache_kick_upload_minio_multipart(ctx, upload_length); - } - } - } - break; - - case PUT_STATE_PART: - if(res != CURLE_OK || res_code!=200L) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_CURL); - } - if(ctx->fail_state) - { - if(cache_cancel_upload_minio(ctx)) - { - ctx->put.state = PUT_STATE_CANCEL; - } - else if(ctx->put.close_state) - { - tango_cache_ctx_destroy(ctx); - } - } - else if(ctx->put.close_state) - { - do_tango_cache_update_end(ctx, true); - } - else - { - size_t upload_length = evbuffer_get_length(ctx->put.evbuf); - if(upload_length >= ctx->instance->param->upload_block_size) - { - cache_kick_upload_minio_multipart(ctx, upload_length); - } - } - break; - - case PUT_STATE_CANCEL: //�ȴ��ر� - if(ctx->put.close_state) - { - tango_cache_ctx_destroy(ctx); - } - break; - - case PUT_STATE_END: - if(res != CURLE_OK || res_code!=200L) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_CURL); - } - if(ctx->instance->param->object_store_way!=CACHE_ALL_HOS && !ctx->fail_state) - { - redis_put_minio_object_meta(ctx, true); - } - else - { - tango_cache_ctx_destroy(ctx); - } - break; - default: break; - } -} - -int http_put_complete_part_data(struct tango_cache_ctx *ctx, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, bool callback) -{ - UNUSED CURLMcode rc; - char minio_url[256], buffer[256]; - - if(NULL == (ctx->curl=curl_easy_init())) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_CURL); - tango_cache_ctx_destroy(ctx, callback); - if(way == PUT_MEM_FREE) free((void *)data); - ctx->instance->statistic.memory_used -= size; - return -1; - } - ctx->put.state = PUT_STATE_END; - - snprintf(minio_url, 256, "http://%s/%s", ctx->hostaddr, ctx->object_key); - curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); - curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_any_cb); - curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); - sprintf(buffer, "token: %s", ctx->instance->param->cache_token); - ctx->headers = curl_slist_append(ctx->headers, buffer); - curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); - curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error); - - if(way == PUT_MEM_COPY) - { - ctx->put.once_request.buff = (char *)malloc(size); - memcpy(ctx->put.once_request.buff, data, size); - } - else - { - ctx->put.once_request.buff = (char *)data; - } - ctx->put.once_request.size = size; - ctx->put.once_request.len = 0; - curl_easy_setopt(ctx->curl, CURLOPT_UPLOAD, 1L); - curl_easy_setopt(ctx->curl, CURLOPT_INFILESIZE, ctx->put.once_request.size); - curl_easy_setopt(ctx->curl, CURLOPT_READFUNCTION, curl_put_once_send_cb); - curl_easy_setopt(ctx->curl, CURLOPT_READDATA, ctx); - - rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl); - assert(rc==CURLM_OK); - return 0; -} - -int do_tango_cache_upload_once_data(struct tango_cache_ctx *ctx, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, bool callback) -{ - ctx->instance->statistic.put_recv_num += 1; - ctx->instance->statistic.memory_used += size; - ctx->instance->error_code = CACHE_OK; - - if(ctx->locate == OBJECT_IN_HOS) - { - return http_put_complete_part_data(ctx, way, data, size, false); - } - else - { - return redis_put_complete_part_data(ctx, way, data, size, false); - } -} - -int do_tango_cache_upload_once_evbuf(struct tango_cache_ctx *ctx, enum EVBUFFER_COPY_WAY way, struct evbuffer *evbuf, bool callback) -{ - size_t size; - - ctx->instance->statistic.put_recv_num += 1; - ctx->instance->error_code = CACHE_OK; - - if(way == EVBUFFER_MOVE) - { - if(evbuffer_add_buffer(ctx->put.evbuf, evbuf)) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_EVBUFFER); - tango_cache_ctx_destroy(ctx, callback); - return -1; - } - } - else - { - if(evbuffer_add_buffer_reference(ctx->put.evbuf, evbuf)) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_EVBUFFER); - tango_cache_ctx_destroy(ctx, callback); - return -1; - } - } - size = evbuffer_get_length(ctx->put.evbuf); - ctx->instance->statistic.memory_used += size; - - if(ctx->locate == OBJECT_IN_HOS) - { - return http_put_complete_part_evbuf(ctx, callback); - } - else - { - return redis_put_complete_part_evbuf(ctx, size, callback); - } -} - -void tango_cache_curl_del_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code) -{ - if(res!=CURLE_OK || (res_code!=204L && res_code!=200L )) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_CURL); - } - tango_cache_ctx_destroy(ctx); -} - -void tango_cache_curl_muldel_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code) -{ - u_int32_t errnum=0; - - if(res!=CURLE_OK || (res_code!=204L && res_code!=200L )) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_CURL); - ctx->del.fail_num = ctx->del.succ_num; - ctx->del.succ_num = 0; - } - else - { - if(!parse_multidelete_xml(ctx->response.buff, ctx->response.len, &errnum, ctx->error, CURL_ERROR_SIZE)) - { - ctx->del.fail_num = ctx->del.succ_num; - ctx->del.succ_num = 0; - } - else - { - ctx->del.fail_num = errnum; - ctx->del.succ_num -= errnum; - } - if(ctx->del.fail_num > 0) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_CURL); - } - } - tango_cache_ctx_destroy(ctx); -} - -int do_tango_cache_multi_delete(struct tango_cache_ctx *ctx, bool callback) -{ - UNUSED CURLMcode rc; - char minio_url[256], buffer[256]; - - ctx->instance->statistic.del_recv_num += ctx->del.succ_num; - ctx->instance->error_code = CACHE_OK; - if(NULL == (ctx->curl=curl_easy_init())) - { - tango_cache_set_fail_state(ctx, CACHE_OUTOF_MEMORY); - tango_cache_ctx_destroy(ctx, callback); - return -1; - } - - snprintf(minio_url, 256, "http://%s/%s/?delete", ctx->hostaddr, ctx->instance->param->bucketname); - curl_easy_setopt(ctx->curl, CURLOPT_POST, 1L); - curl_easy_setopt(ctx->curl, CURLOPT_POSTFIELDSIZE, ctx->response.size); //���Content-Length����CURLOPT_COPYPOSTFIELDS֮ǰ���� - curl_easy_setopt(ctx->curl, CURLOPT_COPYPOSTFIELDS, ctx->response.buff); - curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); - sprintf(buffer, "token: %s", ctx->instance->param->cache_token); - ctx->headers = curl_slist_append(ctx->headers, buffer); - curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); - curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_response_body_save_cb); - curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); - curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error); - - rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl); - assert(rc==CURLM_OK); - easy_string_destroy(&ctx->response); - return 0; -} - -bool fetch_header_over_biz(struct tango_cache_ctx *ctx) -{ - if(ctx->get.need_hdrs!=RESPONSE_HDR_ALL) //��Expiresʱ - { - tango_cache_set_fail_state(ctx, CACHE_ERR_INTERNAL); - ctx->get.state = GET_STATE_DELETE; - promise_failed(ctx->promise, FUTURE_ERROR_CANCEL, tango_cache_get_errstring(ctx)); - return false; - } - - if(ctx->get.response_tag.len > 0) - { - ctx->get.result.data_frag = ctx->get.response_tag.buff; - ctx->get.result.size = ctx->get.response_tag.len; - ctx->get.result.type = RESULT_TYPE_USERTAG; - promise_success(ctx->promise, &ctx->get.result); - easy_string_destroy(&ctx->get.response_tag); - } - if(ctx->response.len > 0) - { - ctx->get.result.data_frag = ctx->response.buff; - ctx->get.result.size = ctx->response.len; - ctx->get.result.type = RESULT_TYPE_HEADER; - promise_success(ctx->promise, &ctx->get.result); - easy_string_destroy(&ctx->response); - } - return true; -} - -static size_t curl_get_response_body_cb(void *ptr, size_t size, size_t count, void *userp) -{ - struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp; - - if(ctx->fail_state || ctx->get.state==GET_STATE_DELETE) - { - return size*count; - } - - if(!fetch_header_over_biz(ctx)) - { - return size*count; - } - - ctx->get.result.data_frag = (const char *)ptr; - ctx->get.result.size = size * count; - ctx->get.result.type = RESULT_TYPE_BODY; - promise_success(ctx->promise, &ctx->get.result); - return size*count; -} - -bool check_expires_fresh_header(struct tango_cache_ctx *ctx) -{ - time_t now_gmt; - - if(ctx->get.need_hdrs != RESPONSE_HDR_ALL) - return true; - - now_gmt = get_gmtime_timestamp(time(NULL)); - - if(now_gmt > ctx->get.expires) - { - tango_cache_set_fail_state(ctx, CACHE_TIMEOUT); - ctx->get.state = GET_STATE_DELETE; //����ʧЧʱ���������ʱ����ɾ������ - ctx->get.result.type = RESULT_TYPE_MISS; - promise_success(ctx->promise, &ctx->get.result); - promise_finish(ctx->promise); - easy_string_destroy(&ctx->response); - return false; - } - - if(ctx->get.last_modify+ctx->get.max_age > now_gmt || now_gmt+ctx->get.min_fresh>ctx->get.expires) - { - tango_cache_set_fail_state(ctx, CACHE_TIMEOUT); - ctx->get.result.type = RESULT_TYPE_MISS; - promise_success(ctx->promise, &ctx->get.result); - promise_finish(ctx->promise); - easy_string_destroy(&ctx->response); - return false; - } - return true; -} - -static bool check_get_result_code(struct tango_cache_ctx *ctx, CURLcode code, long res_code) -{ - if(code != CURLE_OK) - { - tango_cache_set_fail_state(ctx, CACHE_ERR_CURL); - promise_failed(ctx->promise, FUTURE_ERROR_CANCEL, tango_cache_get_errstring(ctx)); - return false; - } - - if(res_code != 200L) - { - if(res_code == 404L) - { - tango_cache_set_fail_state(ctx, CACHE_CACHE_MISS); - ctx->get.result.type = RESULT_TYPE_MISS; - promise_success(ctx->promise, &ctx->get.result); - promise_finish(ctx->promise); - } - else - { - tango_cache_set_fail_state(ctx, CACHE_ERR_INTERNAL); - promise_failed(ctx->promise, FUTURE_ERROR_CANCEL, tango_cache_get_errstring(ctx)); - } - return false; - } - return true; -} - -static size_t curl_get_response_header_cb(void *ptr, size_t size, size_t count, void *userp) -{ - struct tango_cache_ctx *ctx = (struct tango_cache_ctx *)userp; - char *start=(char *)ptr, *pos_colon; - size_t raw_len = size*count, hdrlen=size*count; - char usertag[2048]; - size_t datalen; - - if(ctx->fail_state || ctx->get.state==GET_STATE_DELETE) - { - return raw_len; - } - if(ctx->res_code == 0) //�״�Ӧ��ʱ�ȿ�Ӧ�����Ƿ���200 - { - UNUSED CURLcode code = curl_easy_getinfo(ctx->curl, CURLINFO_RESPONSE_CODE, &ctx->res_code); - if(!check_get_result_code(ctx, code, ctx->res_code)) - { - return raw_len; - } - ctx->get.result.location = OBJECT_IN_HOS; - } - pos_colon = (char*)memchr(start, ':', raw_len); - if(pos_colon == NULL) - { - return raw_len; - } - - datalen = pos_colon - start; - switch(datalen) - { - case 7: - if(strcmp_one_word_mesa_equal_len("expires", "EXPIRES", start, 7)) - { - ctx->get.need_hdrs |= RESPONSE_HDR_EXPIRES; - ctx->get.expires = expires_hdr2timestamp(pos_colon + 1, raw_len - datalen - 1); - if(!check_expires_fresh_header(ctx)) - { - return raw_len; - } - } - break; - case 13: - if(strcmp_one_word_mesa_equal_len("x-amz-meta-lm", "X-AMZ-META-LM", start, 13)) - { - ctx->get.need_hdrs |= RESPONSE_HDR_LAST_MOD; - sscanf(pos_colon+1, "%lu", &ctx->get.last_modify); - if(!check_expires_fresh_header(ctx)) - { - return raw_len; - } - } - break; - case 15: - if(strcmp_one_word_mesa_equal_len("x-amz-meta-user", "X-AMZ-META-USER", start, 15)) - { - if((hdrlen = Base64_DecodeBlock((unsigned char*)pos_colon+1, raw_len-datalen-1, (unsigned char*)usertag, 2048))>0) - { - easy_string_savedata(&ctx->get.response_tag, usertag, hdrlen); - } - } - break; - case 14: - if(strcmp_one_word_mesa_equal_len("content-length", "CONTENT-LENGTH", start, 14)) - { - sscanf(pos_colon+1, "%lu", &ctx->get.result.tlength); - } - break; - case 11: if(strcmp_one_word_mesa_equal_len("content-md5", "CONTENT-MD5", start, 11)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break; - case 12: if(strcmp_one_word_mesa_equal_len("content-type", "CONTENT-TYPE", start, 12)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break; - case 16: if(strcmp_one_word_mesa_equal_len("content-encoding", "CONTENT-ENCODING", start, 16)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break; - case 19: if(strcmp_one_word_mesa_equal_len("content-disposition", "CONTENT-DISPOSITION", start, 19)) easy_string_savedata(&ctx->response, (const char*)ptr, raw_len); break; - default: break; - } - return raw_len; -} - -void tango_cache_curl_get_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code) -{ - switch(ctx->get.state) - { - case GET_STATE_START: - if(!ctx->fail_state && check_get_result_code(ctx, res, res_code)) - { - if(ctx->method!=CACHE_REQUEST_HEAD || fetch_header_over_biz(ctx)) //HEAD���ֵ��ֶβ�ȫ�Ȳ�ɾ������������ޣ� - { - ctx->get.result.type = RESULT_TYPE_END; - promise_success(ctx->promise, &ctx->get.result); - promise_finish(ctx->promise); - } - } - tango_cache_ctx_destroy(ctx); - break; - - case GET_STATE_DELETE: - ctx->get.state = GET_STATE_END; - cache_delete_minio_object(ctx); - break; - - case GET_STATE_END: - tango_cache_ctx_destroy(ctx); - break; - default: assert(0);break; - } -} - -static int tango_cache_fetch_minio(struct tango_cache_ctx *ctx) -{ - UNUSED CURLMcode rc; - char minio_url[256], buffer[256]; - - if(NULL == (ctx->curl=curl_easy_init())) - { - tango_cache_ctx_destroy(ctx); - return -1; - } - - snprintf(minio_url, 256, "http://%s/%s", ctx->hostaddr, ctx->object_key); - curl_easy_setopt(ctx->curl, CURLOPT_URL, minio_url); - if(ctx->method == CACHE_REQUEST_HEAD) - { - curl_easy_setopt(ctx->curl, CURLOPT_NOBODY, 1L); - } - curl_easy_setopt(ctx->curl, CURLOPT_WRITEFUNCTION, curl_get_response_body_cb); - curl_easy_setopt(ctx->curl, CURLOPT_WRITEDATA, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_PRIVATE, ctx); - curl_easy_setopt(ctx->curl, CURLOPT_HEADERFUNCTION, curl_get_response_header_cb); - curl_easy_setopt(ctx->curl, CURLOPT_HEADERDATA, ctx); - sprintf(buffer, "token: %s", ctx->instance->param->cache_token); - ctx->headers = curl_slist_append(ctx->headers, buffer); - curl_easy_setopt(ctx->curl, CURLOPT_HTTPHEADER, ctx->headers); - curl_set_common_options(ctx->curl, ctx->instance->param->transfer_timeout, ctx->error); - - rc = curl_multi_add_handle(ctx->instance->multi_hd, ctx->curl); - assert(rc==CURLM_OK); - return 1; -} - -static void redis_redirect_object2minio_cb(struct tango_cache_ctx *ctx) -{ - struct promise *p = ctx->promise; - - ctx->get.state = GET_STATE_START; - ctx->locate = OBJECT_IN_HOS; - if(ctx->instance->statistic.session_http>=ctx->instance->param->maximum_sessions) - { - tango_cache_set_fail_state(ctx, CACHE_OUTOF_MEMORY); - promise_failed(p, FUTURE_ERROR_CANCEL, tango_cache_get_errstring(ctx)); - tango_cache_ctx_destroy(ctx); - } - else if(tango_cache_fetch_minio(ctx) != 1) - { - promise_failed(p, FUTURE_ERROR_CANCEL, "tango_cache_fetch_minio failed"); - } -} - -int do_tango_cache_fetch_object(struct tango_cache_ctx *ctx, enum OBJECT_LOCATION where_to_get) -{ - ctx->instance->statistic.get_recv_num += 1; - switch(where_to_get) - { - case OBJECT_IN_HOS: - ctx->locate = OBJECT_IN_HOS; - return (tango_cache_fetch_minio(ctx)==1)?0:-2; - case OBJECT_IN_REDIS: - ctx->locate = OBJECT_IN_REDIS; - return tango_cache_fetch_redis(ctx); - default: - ctx->get.redis_redirect_minio_cb = redis_redirect_object2minio_cb; - return tango_cache_try_fetch_redis(ctx); - } - return 0; -} - -int do_tango_cache_head_object(struct tango_cache_ctx *ctx, enum OBJECT_LOCATION where_to_head) -{ - ctx->instance->statistic.get_recv_num += 1; - if(where_to_head == OBJECT_IN_REDIS) - { - return tango_cache_head_redis(ctx); - } - else - { - return (tango_cache_fetch_minio(ctx)==1)?0:-1; - } -} - diff --git a/cache/src/tango_cache_transfer.h b/cache/src/tango_cache_transfer.h deleted file mode 100644 index ec17215..0000000 --- a/cache/src/tango_cache_transfer.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef __TANGO_CACHE_UPLOAD_H__ -#define __TANGO_CACHE_UPLOAD_H__ - -#include <sys/queue.h> -#include <curl/curl.h> - -#include "tango_cache_client_in.h" - -bool check_expires_fresh_header(struct tango_cache_ctx *ctx); -bool fetch_header_over_biz(struct tango_cache_ctx *ctx); - -void tango_cache_curl_put_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code); -void tango_cache_curl_get_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code); -void tango_cache_curl_del_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code); -void tango_cache_curl_muldel_done(struct tango_cache_ctx *ctx, CURLcode res, long res_code); - -int cache_delete_minio_object(struct tango_cache_ctx *ctx, bool call_back=false); -int do_tango_cache_multi_delete(struct tango_cache_ctx *ctx, bool callback=false); - -bool cache_cancel_upload_minio(struct tango_cache_ctx *ctx); -bool cache_kick_upload_minio_multipart(struct tango_cache_ctx *ctx, size_t block_len); -int do_tango_cache_update_end(struct tango_cache_ctx *ctx, bool callback); - -int do_tango_cache_upload_once_data(struct tango_cache_ctx *ctx, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, bool call_back=false); -int do_tango_cache_upload_once_evbuf(struct tango_cache_ctx *ctx, enum EVBUFFER_COPY_WAY way, struct evbuffer *evbuf, bool call_back=false); - -int do_tango_cache_head_object(struct tango_cache_ctx *ctx, enum OBJECT_LOCATION where_to_head); -int do_tango_cache_fetch_object(struct tango_cache_ctx *ctx, enum OBJECT_LOCATION where_to_get); - -#endif - diff --git a/cache/src/tango_cache_xml.cpp b/cache/src/tango_cache_xml.cpp deleted file mode 100644 index 97280b6..0000000 --- a/cache/src/tango_cache_xml.cpp +++ /dev/null @@ -1,172 +0,0 @@ -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> -#include <fcntl.h> - -#include <libxml/parser.h> -#include "tango_cache_xml.h" - -bool parse_uploadID_xml(const char *content, int len, char **uploadID) -{ - xmlDoc *pdoc; - xmlNode *pcur; - - if((pdoc = xmlParseMemory(content, len)) == NULL) - { - return false; - } - if((pcur = xmlDocGetRootElement(pdoc)) == NULL) - { - xmlFreeDoc(pdoc); - return false; - } - - while(pcur->type != XML_ELEMENT_NODE) - pcur = pcur->next; - if(xmlStrcmp(pcur->name, (const xmlChar *)"InitiateMultipartUploadResult")) - { - xmlFreeDoc(pdoc); - return false; - } - pcur = pcur->children; - while(pcur != NULL) - { - if(pcur->type != XML_ELEMENT_NODE || xmlStrcmp(pcur->name, (const xmlChar *)"UploadId")) - { - pcur = pcur->next; - continue; - } - *uploadID = (char *)xmlNodeGetContent(pcur); - xmlFreeDoc(pdoc); - return true; - } - - xmlFreeDoc(pdoc); - return false; -} - -void construct_complete_xml(struct tango_cache_ctx *ctx, char **xml, int *len) -{ - struct multipart_etag_list *etag; - xmlDoc *pdoc; - xmlNode *root, *child; - char number[20]; - - pdoc = xmlNewDoc((const xmlChar *)"1.0"); - root = xmlNewNode(NULL, (const xmlChar *)"CompleteMultipartUpload"); - /*Big data deletion of this field parsing, shielding this field**/ - //xmlNewProp(root, (const xmlChar *)"xmlns",(const xmlChar *)"http://s3.amazonaws.com/doc/2006-03-01/"); - xmlDocSetRootElement(pdoc, root); - - TAILQ_FOREACH(etag, &ctx->put.etag_head, node) - { - sprintf(number, "%u", etag->part_number); - child = xmlNewChild(root, NULL, (const xmlChar*)"Part", NULL); - xmlNewChild(child, NULL, (const xmlChar*)"ETag", (const xmlChar*)etag->etag); - xmlNewChild(child, NULL, (const xmlChar*)"PartNumber", (const xmlChar*)number); - } - - xmlDocDumpFormatMemory(pdoc, (xmlChar **)xml, len, 1); - xmlFreeDoc(pdoc); -} - -static void fill_multidelete_xml_errcode(xmlNode *error, char *out, int size) -{ - xmlChar *errcode; - xmlNode *child = error->children; - - while(child != NULL) - { - if(child->type != XML_ELEMENT_NODE || xmlStrcmp(child->name, (const xmlChar *)"Message")) - { - child = child->next; - continue; - } - errcode = xmlNodeGetContent(child); - snprintf(out, size, "%s", (char *)errcode); - xmlFree(errcode); - break; - } -} - -bool parse_multidelete_xml(const char *xml, int len, u_int32_t *errnum, char *errstr, int size) -{ - xmlDoc *pdoc; - xmlNode *pcur; - int errornum=0; - - if((pdoc = xmlParseMemory(xml, len)) == NULL) - { - return false; - } - if((pcur = xmlDocGetRootElement(pdoc)) == NULL) - { - xmlFreeDoc(pdoc); - return false; - } - - while(pcur->type != XML_ELEMENT_NODE) - pcur = pcur->next; - if(xmlStrcmp(pcur->name, (const xmlChar *)"DeleteResult")) - { - xmlFreeDoc(pdoc); - return false; - } - pcur = pcur->children; - while(pcur != NULL) - { - if(pcur->type != XML_ELEMENT_NODE || xmlStrcmp(pcur->name, (const xmlChar *)"Error")) - { - pcur = pcur->next; - continue; - } - if(errornum == 0) - { - fill_multidelete_xml_errcode(pcur, errstr, size); - } - errornum++; - pcur = pcur->next; - } - *errnum = errornum; - - xmlFreeDoc(pdoc); - return true; -} - -void construct_multiple_delete_xml(const char *bucket, char *key[], u_int32_t num, int is_hash, char **xml, size_t *len) -{ - xmlDoc *pdoc; - xmlNode *root, *child; - int xmllen; - - pdoc = xmlNewDoc((const xmlChar *)"1.0"); - root = xmlNewNode(NULL, (const xmlChar *)"Delete"); - xmlDocSetRootElement(pdoc, root); - - xmlNewChild(root, NULL, (const xmlChar*)"Quiet", (const xmlChar*)"true"); - - if(is_hash) - { - char hashkey[72], sha256[72]; - for(u_int32_t i=0; i<num; i++) - { - child = xmlNewChild(root, NULL, (const xmlChar*)"Object", NULL); - caculate_sha256(key[i], strlen(key[i]), sha256, 72); - snprintf(hashkey, 256, "%c%c_%c%c_%s", sha256[0], sha256[1], sha256[2], sha256[3], sha256+4); - xmlNewChild(child, NULL, (const xmlChar*)"Key", (const xmlChar*)hashkey); - } - } - else - { - for(u_int32_t i=0; i<num; i++) - { - child = xmlNewChild(root, NULL, (const xmlChar*)"Object", NULL); - xmlNewChild(child, NULL, (const xmlChar*)"Key", (const xmlChar*)key[i]); - } - } - xmlDocDumpFormatMemoryEnc(pdoc, (xmlChar **)xml, &xmllen, "UTF-8", 1); - xmlFreeDoc(pdoc); - *len = xmllen; -} - diff --git a/cache/src/tango_cache_xml.h b/cache/src/tango_cache_xml.h deleted file mode 100644 index 43543dc..0000000 --- a/cache/src/tango_cache_xml.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef __TANGO_CACHE_XML_H__ -#define __TANGO_CACHE_XML_H__ - -#include "tango_cache_client_in.h" - -bool parse_uploadID_xml(const char *content, int len, char **uploadID); -void construct_complete_xml(struct tango_cache_ctx *ctx, char **xml, int *len); -void construct_multiple_delete_xml(const char *bucket, char *key[], u_int32_t num, int is_hash, char **xml, size_t *len); -bool parse_multidelete_xml(const char *xml, int len, u_int32_t *errnum, char *errstr, int size); - -#endif - diff --git a/cache/test/CMakeLists.txt b/cache/test/CMakeLists.txt deleted file mode 100644 index f2a59a3..0000000 --- a/cache/test/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -add_definitions(-fPIC -Wall -g) - -add_executable (cache_evbase_test cache_evbase_test.cpp) -target_include_directories(cache_evbase_test PRIVATE ../include) -target_link_libraries(cache_evbase_test common) -target_link_libraries(cache_evbase_test tango-cache-client libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson) -target_link_libraries (cache_evbase_test MESA_handle_logger MESA_htable MESA_prof_load MESA_field_stat wiredLB pthread z) - -add_executable (cache_evbase_benchmark cache_evbase_benchmark.cpp) -target_link_libraries(cache_evbase_benchmark tango-cache-client libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson) -target_link_libraries (cache_evbase_benchmark MESA_handle_logger MESA_htable MESA_prof_load MESA_field_stat wiredLB pthread z) - -#add_executable (cache_evbase_test_threads cache_evbase_test_threads.cpp) -#target_link_libraries(cache_evbase_test_threads tango-cache-client libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson) -#target_link_libraries (cache_evbase_test_threads MESA_handle_logger MESA_htable MESA_prof_load wiredLB pthread z) - -add_executable (tango_cache_test tango_cache_test.cpp) -target_link_libraries(tango_cache_test tango-cache-client libevent-static openssl-crypto-static openssl-ssl-static libxml2-static libcurl-static hiredis-static cjson) -target_link_libraries (tango_cache_test MESA_handle_logger MESA_htable MESA_prof_load MESA_field_stat wiredLB pthread) - -#INSTALL (TARGETS cache_evbase_test cache_evbase_test_threads tango_cache_test cache_evbase_benchmark DESTINATION bin) -INSTALL (TARGETS cache_evbase_test tango_cache_test cache_evbase_benchmark DESTINATION bin) -INSTALL (FILES ${CMAKE_CURRENT_SOURCE_DIR}/pangu_tg_cahce.conf DESTINATION bin) -INSTALL (FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmd.txt DESTINATION bin) diff --git a/cache/test/Makefile b/cache/test/Makefile deleted file mode 100644 index 9068db8..0000000 --- a/cache/test/Makefile +++ /dev/null @@ -1,39 +0,0 @@ -CCC=g++ - -INC_PATH=-I../include -I../include/libevent2 -CFLAGS=-Wall -g $(INC_PATH) - -LIBS = -lMESA_handle_logger -lMESA_prof_load -lWiredLB -LIBS += -lssl -lcrypto -LIBS += ../lib/libtango_cache_client.a -LIBS += ./lib/libcurl.a ./lib/libevent.a ./lib/libxml2.a ./lib/libhiredis_vip.a - -OBJS = tango_cache_test.o -OBJS_EVBASE=cache_evbase_test.o -OBJS_EVBASE_BENCHMARK=cache_evbase_benchmark.o - -TARGET_EXE=tango_cache_test -TARGET_EXE_EVBASE=cache_evbase_test -TARGET_EXE_EVBASE_BENCHMARK=cache_evbase_benchmark - -ALL:$(TARGET_EXE) $(TARGET_EXE_EVBASE) $(TARGET_EXE_EVBASE_BENCHMARK) - -$(TARGET_EXE):$(OBJS) - $(CCC) $(LDFLAGS) $^ -o $@ $(LIBS) - -$(TARGET_EXE_EVBASE):$(OBJS_EVBASE) - $(CCC) $(LDFLAGS) $^ -o $@ $(LIBS) -lpthread - -$(TARGET_EXE_EVBASE_BENCHMARK):$(OBJS_EVBASE_BENCHMARK) - $(CCC) $(LDFLAGS) $^ -o $@ $(LIBS) -lpthread - -.c.o: - $(CCC) $(CFLAGS) -c $< -.cpp.o: - $(CCC) $(CFLAGS) -c $< - --include $(DEPS) - -clean: - rm -rf $(OBJS) $(TARGET_EXE) $(OBJS_EVBASE) $(TARGET_EXE_EVBASE) $(OBJS_EVBASE_BENCHMARK) $(TARGET_EXE_EVBASE_BENCHMARK) - diff --git a/cache/test/README.txt b/cache/test/README.txt deleted file mode 100644 index 7afcf75..0000000 --- a/cache/test/README.txt +++ /dev/null @@ -1,16 +0,0 @@ -1��tango_cache_test.c���칷ʹ�õ�demo����Ҫ�ⲿ����������event_base��ִ�з������£� - ��һ���նˣ�ִ��./tango_cache_test�� - ����һ���նˣ�ִ�У� - ��ȡ��echo GET:tcpdump_all_1.pcap > cache.fifo - ��ʽ�ϴ��� -echo PUT:tcpdump_all_1.pcap > cache.fifo - �����ϴ��� -echo PUTONCE:tcpdump_all_1.pcap > cache.fifo - - -2��cache_evbase_test.c�Ƕ��칷cache�ķ�װ���û������Լ�����event_base�������ڲ�������ִ�з������£� - ��ȡ��./cache_evbase_test GET:tcpdump_all_1.pcap - ��ʽ�ϴ���./cache_evbase_test PUT:tcpdump_all_1.pcap - �����ϴ���./cache_evbase_test PUTONCE:tcpdump_all_1.pcap - -�������ж�������ļ�������tcpdump_all_1.pcap���Եģ�ʵ��Ӧ��ʱ�滻�ɾ�����ļ�����
\ No newline at end of file diff --git a/cache/test/cache_evbase_benchmark.cpp b/cache/test/cache_evbase_benchmark.cpp deleted file mode 100644 index 6e50d02..0000000 --- a/cache/test/cache_evbase_benchmark.cpp +++ /dev/null @@ -1,444 +0,0 @@ -#include <sys/ioctl.h> -#include <sys/socket.h> -#include <netinet/in.h> -#include <arpa/inet.h> -#include <net/if.h> -#include <unistd.h> -#include <stdio.h> -#include <stdlib.h> -#include <assert.h> -#include <errno.h> -#include <sys/prctl.h> -#include <sys/un.h> -#include <unistd.h> -#include <sys/poll.h> -#include <curl/curl.h> -#include <fcntl.h> -#include <sys/stat.h> -#include <errno.h> -#include <sys/cdefs.h> -#include <MESA/MESA_handle_logger.h> -#include <pthread.h> -#include <sys/prctl.h> - -#include "object_store_client.h" - -#define METHOD_GET 1 -#define METHOD_PUT 2 -#define METHOD_HEAD 3 -#define METHOD_PUTONCEEV 4 -#define METHOD_PUTONCE 5 -#define METHOD_DEL 6 - -struct object_store_instance *instance_asyn; - -struct filecontent -{ - char *buf; - size_t len; -}; - -struct future_pdata -{ - struct future * future; - FILE *fp; - char filename[256]; -}; - -void get_future_success(future_result_t* result, void * user) -{ - struct tango_cache_result *res = cache_evbase_read_result(result); - struct future_pdata *pdata = (struct future_pdata *)user; - //char buffer[1024]; - - switch(res->type) - { - case RESULT_TYPE_USERTAG: - case RESULT_TYPE_HEADER: - //memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size); - //buffer[res->size] = '\0'; - //printf("%s", buffer); - break; - case RESULT_TYPE_BODY: - //fwrite(res->data_frag, res->size, 1, pdata->fp); - break; - case RESULT_TYPE_MISS: - //printf("cache not hit/fresh\n"); - case RESULT_TYPE_END: - //if(res->type != RESULT_TYPE_MISS) - // printf("get cache over, total length: %ld\n", res->tlength); - future_destroy(pdata->future); - //fclose(pdata->fp); - free(pdata); - break; - default:break; - } -} - -void get_future_failed(enum e_future_error err, const char * what, void * user) -{ - struct future_pdata *pdata = (struct future_pdata *)user; - future_destroy(pdata->future); - free(pdata); - //printf("GET fail: %s\n", what); -} - -void head_future_success(future_result_t* result, void * user) -{ - struct tango_cache_result *res = cache_evbase_read_result(result); - struct future_pdata *pdata = (struct future_pdata *)user; - char buffer[1024]; - - switch(res->type) - { - case RESULT_TYPE_USERTAG: - case RESULT_TYPE_HEADER: - memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size); - buffer[res->size] = '\0'; - printf("%s", buffer); - break; - case RESULT_TYPE_BODY: - assert(0); - break; - case RESULT_TYPE_MISS: - printf("cache not hit/fresh\n"); - case RESULT_TYPE_END: - if(res->type != RESULT_TYPE_MISS) - printf("HEAD cache over, total length: %ld\n", res->tlength); - future_destroy(pdata->future); - free(pdata); - break; - default:break; - } -} - -void head_future_failed(enum e_future_error err, const char * what, void * user) -{ - printf("HEAD fail: %s\n", what); -} - - -void put_future_success(future_result_t* result, void * user) -{ - struct future_pdata *pdata = (struct future_pdata *)user; - - //printf("PUT %s succ\n", pdata->filename); - future_destroy(pdata->future); - free(pdata); -} -void put_future_failed(enum e_future_error err, const char * what, void * user) -{ - struct future_pdata *pdata = (struct future_pdata *)user; - - //printf("PUT %s fail: %s\n", what, pdata->filename); - future_destroy(pdata->future); - free(pdata); -} - -void del_future_success(future_result_t* result, void * user) -{ - struct future_pdata *pdata = (struct future_pdata *)user; - - printf("DEL %s succ\n", pdata->filename); - future_destroy(pdata->future); - free(pdata); -} -void del_future_failed(enum e_future_error err, const char * what, void * user) -{ - struct future_pdata *pdata = (struct future_pdata *)user; - - printf("DEL %s fail: %s\n", pdata->filename, what); - future_destroy(pdata->future); - free(pdata); -} - -char * get_file_content(const char *filename, size_t *filelen_out) -{ - char *buffer; - FILE *fp; - size_t filelen = 0; - struct stat filestat; - int readlen; - - fp = fopen(filename, "rb"); - if(fp == NULL) - { - printf("fopen file %s failed.\n", filename); - return NULL; - } - if(fstat(fileno(fp), &filestat)) - { - printf("fstat %s failed.\n", filename); - return NULL; - } - - buffer = (char *)malloc(filestat.st_size); - - while(filelen < (size_t)filestat.st_size) - { - readlen = fread(buffer + filelen, 1, filestat.st_size - filelen, fp); - if(readlen < 0) - { - printf("read error: %s\n", strerror(errno)); - return NULL; - } - - filelen += readlen; - } - fclose(fp); - *filelen_out = filestat.st_size; - - return buffer; -} - -struct cache_statistics g_out_last; -void timer_cb(evutil_socket_t fd, short what, void *arg) -{ - struct timeval tv; - struct cache_statistics out_now; - struct cache_statistics out; - - tv.tv_sec = 10; - tv.tv_usec = 0; - - /*static int num=0; - num++; - if(ctx_global!=NULL && num>5) - { - tango_cache_update_end(ctx_global); - ctx_global = NULL; - }*/ - - object_store_get_statistics(instance_asyn, &out_now); - out.del_error_num = out_now.del_error_num - g_out_last.del_error_num; - out.del_recv_num = out_now.del_recv_num - g_out_last.del_recv_num; - out.del_succ_num = out_now.del_succ_num - g_out_last.del_succ_num; - out.get_err_http = out_now.get_err_http - g_out_last.get_err_http; - out.get_err_redis = out_now.get_err_redis - g_out_last.get_err_redis; - out.get_miss_num = out_now.get_miss_num - g_out_last.get_miss_num; - out.get_recv_num = out_now.get_recv_num - g_out_last.get_recv_num; - out.get_succ_http = out_now.get_succ_http - g_out_last.get_succ_http; - out.get_succ_redis = out_now.get_succ_redis - g_out_last.get_succ_redis; - out.put_err_http = out_now.put_err_http - g_out_last.put_err_http; - out.put_err_redis = out_now.put_err_redis - g_out_last.put_err_redis; - out.put_recv_num = out_now.put_recv_num - g_out_last.put_recv_num; - out.put_succ_http = out_now.put_succ_http - g_out_last.put_succ_http; - out.put_succ_redis = out_now.put_succ_redis - g_out_last.put_succ_redis; - out.session_http = out_now.session_http; - out.session_redis = out_now.session_redis; - out.memory_used = out_now.memory_used; - out.totaldrop_num = out_now.totaldrop_num - g_out_last.totaldrop_num; - - printf("-------------------------------------------------------------------------------------------\n" - "get_recv: %llu, get_http: %llu, get_redis: %llu, get_fail_http: %llu, get_fail_redis: %llu, get_miss: %llu\n" - "put_recv: %llu, put_http: %llu, put_redis: %llu, put_fail_http: %llu, put_fail_redis: %llu\n" - "del_recv: %llu, del_succ: %llu, del_fail: %llu, drop_num: %llu, session_redis: %llu, session_http: %llu, memory: %llu\n", - out.get_recv_num, out.get_succ_http, out.get_succ_redis, out.get_err_http, out.get_err_redis, out.get_miss_num, - out.put_recv_num, out.put_succ_http, out.put_succ_redis, out.put_err_http, out.put_err_redis, - out.del_recv_num, out.del_succ_num, out.del_error_num, out.totaldrop_num, out.session_redis, out.session_http, out.memory_used); - - g_out_last = out_now; - event_add((struct event *)arg, &tv); -} - -struct filecontentcmd -{ - int method; - int threads; - int total_num; - int sess_limit; - char file[256]; -}; - - -static void* thread_transfer_cmd(void *arg) -{ - int index=0; - char filename_in[256]; - struct tango_cache_meta_put putmeta; - struct tango_cache_meta_get getmeta; - struct future_pdata *pdata; - struct cache_evbase_ctx *ctx; - struct filecontent filecont; - size_t remain_len; - struct cache_statistics out; - struct evbuffer *evbuf; - struct filecontentcmd *filecmd = (struct filecontentcmd *)arg; - - prctl(PR_SET_NAME, "transfer_cmd"); - - memset(&putmeta, 0, sizeof(struct tango_cache_meta_put)); - memset(&getmeta, 0, sizeof(struct tango_cache_meta_get)); - putmeta.std_hdr[HDR_CONTENT_TYPE] = "Content-Type: maintype/subtype"; - putmeta.std_hdr[HDR_CONTENT_ENCODING] = "Content-Encoding: gzip"; - putmeta.usertag = "Etag: hgdkqkwdwqekdfjwjfjwelkjfkwfejwhf\r\n"; - putmeta.usertag_len = strlen(putmeta.usertag); - - filecont.buf = get_file_content(filecmd->file, &filecont.len); - assert(filecont.buf != NULL); - - while(1) - { - object_store_get_statistics(instance_asyn, &out); - if(out.session_http >= filecmd->sess_limit || out.session_redis>=filecmd->sess_limit) - { - usleep(1000); - continue; - } - - switch(filecmd->method) - { - case METHOD_GET: - sprintf(filename_in, "%s_%u", filecmd->file, index); - getmeta.url = filename_in; - pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata)); - pdata->future = future_create("_get", get_future_success, get_future_failed, (void *)pdata); - object_store_fetch_object(instance_asyn, pdata->future, &getmeta, OBJECT_IN_UNKNOWN); - break; - - case METHOD_PUT: - remain_len = filecont.len; - - sprintf(filename_in, "%s_%u", filecmd->file, index); - putmeta.url = filename_in; - - pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata)); - pdata->future = future_create("_put", put_future_success, put_future_failed, (void *)pdata); - ctx = object_store_update_start(instance_asyn, pdata->future, &putmeta); - if(ctx == NULL) - { - future_destroy(pdata->future); - free(pdata); - continue; - } - while(remain_len >= 1024) - { - object_store_update_frag_data(ctx, PUT_MEM_COPY, filecont.buf+(filecont.len-remain_len), 1024); - remain_len -= 1024; - } - if(remain_len > 0) - { - object_store_update_frag_data(ctx, PUT_MEM_COPY, filecont.buf+(filecont.len-remain_len), remain_len); - } - if(object_store_update_end(ctx, pdata->filename, 256)) - { - future_destroy(pdata->future); - free(pdata); - } - break; - - case METHOD_HEAD: - pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata)); - pdata->future = future_create("_head", head_future_success, head_future_failed, (void *)pdata); - object_store_head_object(instance_asyn, pdata->future, &getmeta); - break; - - case METHOD_DEL: - pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata)); - pdata->future = future_create("_del", del_future_success, del_future_failed, (void *)pdata); - sprintf(pdata->filename, "%s_%u", filecmd->file, index); - object_store_delete_object(instance_asyn, pdata->future, pdata->filename); - break; - case METHOD_PUTONCE: - remain_len = filecont.len; - - sprintf(filename_in, "%s_%u", filecmd->file, index); - putmeta.url = filename_in; - - pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata)); - pdata->future = future_create("_putonce", put_future_success, put_future_failed, (void *)pdata); - if(object_store_upload_once_data(instance_asyn, pdata->future, PUT_MEM_COPY, filecont.buf, filecont.len, &putmeta, pdata->filename, 256)) - { - future_destroy(pdata->future); - free(pdata); - } - break; - case METHOD_PUTONCEEV: - remain_len = filecont.len; - - sprintf(filename_in, "%s_%u", filecmd->file, index); - putmeta.url = filename_in; - - pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata)); - pdata->future = future_create("_putonceev", put_future_success, put_future_failed, (void *)pdata); - evbuf = evbuffer_new(); - - remain_len = filecont.len; - while(remain_len >= 1024) - { - evbuffer_add(evbuf, filecont.buf+(filecont.len-remain_len), 1024); - remain_len -= 1024; - } - if(remain_len > 0) - { - evbuffer_add(evbuf, filecont.buf+(filecont.len-remain_len), remain_len); - } - if(object_store_upload_once_evbuf(instance_asyn, pdata->future, evbuf, &putmeta, pdata->filename, 256)) - { - future_destroy(pdata->future); - free(pdata); - } - break; - default:break; - } - - index = (index+1) % filecmd->total_num; - } - - return NULL; -} - -int main(int argc, char **argv) -{ - pthread_t thread_tid; - pthread_attr_t attr; - void *runtime_log; - struct filecontentcmd filecmd; - time_t now, remain; - - struct event ev_timer; - struct timeval tv; - struct event_base *ev_base; - - if(argc != 6) - { - printf("USAGE: %s <method,1-GET,2-PUT,5-PUTONCE> <file> <threads> <total_num> <limit_session_num>\n", argv[0]); - return -1; - } - - runtime_log = (void *)MESA_create_runtime_log_handle("./runtime.log", 10); - if(NULL==runtime_log) - { - return -1; - } - filecmd.method = atoi(argv[1]); - filecmd.threads = atoi(argv[3]); - filecmd.total_num = atoi(argv[4]); - filecmd.sess_limit = atoi(argv[5]); - sprintf(filecmd.file, "%s", argv[2]); - - object_store_global_init(); - instance_asyn = object_store_instance_new("./pangu_tg_cahce.conf", "TANGO_CACHE", filecmd.threads, runtime_log); - assert(instance_asyn!=NULL); - - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - for(int i=0; i<4; i++) - { - if(pthread_create(&thread_tid, &attr, thread_transfer_cmd, &filecmd)) - { - return -1; - } - } - - ev_base = event_base_new(); - now = time(NULL); - remain = 10 - (now % 10); - tv.tv_sec = remain; - tv.tv_usec = 0; - evtimer_assign(&ev_timer, ev_base, timer_cb, &ev_timer); - evtimer_add(&ev_timer, &tv); - event_base_dispatch(ev_base); - return 0; -} - diff --git a/cache/test/cache_evbase_test.cpp b/cache/test/cache_evbase_test.cpp deleted file mode 100644 index 0af88e8..0000000 --- a/cache/test/cache_evbase_test.cpp +++ /dev/null @@ -1,333 +0,0 @@ -#include <sys/ioctl.h> -#include <sys/socket.h> -#include <netinet/in.h> -#include <arpa/inet.h> -#include <net/if.h> -#include <unistd.h> -#include <stdio.h> -#include <stdlib.h> -#include <assert.h> -#include <errno.h> -#include <sys/prctl.h> -#include <sys/un.h> -#include <unistd.h> -#include <sys/poll.h> -#include <curl/curl.h> -#include <fcntl.h> -#include <sys/stat.h> -#include <errno.h> -#include <sys/cdefs.h> -#include <MESA/MESA_handle_logger.h> - -#include "cache_evbase_client.h" - -struct cache_evbase_instance *instance_asyn; -int runing_over=0; - -struct future_pdata -{ - struct future * future; - FILE *fp; - char filename[256]; -}; - -void get_future_success(future_result_t* result, void * user) -{ - struct tango_cache_result *res = cache_evbase_read_result(result); - struct future_pdata *pdata = (struct future_pdata *)user; - char buffer[1024]; - - switch(res->type) - { - case RESULT_TYPE_USERTAG: - case RESULT_TYPE_HEADER: - memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size); - buffer[res->size] = '\0'; - printf("%s", buffer); - break; - case RESULT_TYPE_BODY: - fwrite(res->data_frag, res->size, 1, pdata->fp); - break; - case RESULT_TYPE_MISS: - printf("cache not hit/fresh\n"); - case RESULT_TYPE_END: - if(res->type != RESULT_TYPE_MISS) - printf("get cache over, total length: %ld\n", res->tlength); - future_destroy(pdata->future); - fclose(pdata->fp); - free(pdata); - runing_over = 1; - break; - default:break; - } -} - -void get_future_failed(enum e_future_error err, const char * what, void * user) -{ - printf("GET fail: %s\n", what); - runing_over = 2; -} - -void head_future_success(future_result_t* result, void * user) -{ - struct tango_cache_result *res = cache_evbase_read_result(result); - struct future_pdata *pdata = (struct future_pdata *)user; - char buffer[1024]; - - switch(res->type) - { - case RESULT_TYPE_USERTAG: - case RESULT_TYPE_HEADER: - memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size); - buffer[res->size] = '\0'; - printf("%s", buffer); - break; - case RESULT_TYPE_BODY: - assert(0); - break; - case RESULT_TYPE_MISS: - printf("cache not hit/fresh\n"); - case RESULT_TYPE_END: - if(res->type != RESULT_TYPE_MISS) - printf("HEAD cache over, total length: %ld\n", res->tlength); - future_destroy(pdata->future); - free(pdata); - runing_over = 1; - break; - default:break; - } -} - -void head_future_failed(enum e_future_error err, const char * what, void * user) -{ - printf("HEAD fail: %s\n", what); - runing_over = 2; -} - - -void put_future_success(future_result_t* result, void * user) -{ - struct future_pdata *pdata = (struct future_pdata *)user; - - printf("PUT %s succ\n", pdata->filename); - future_destroy(pdata->future); - free(pdata); - runing_over = 1; -} -void put_future_failed(enum e_future_error err, const char * what, void * user) -{ - struct future_pdata *pdata = (struct future_pdata *)user; - - printf("PUT %s fail: %s\n", what, pdata->filename); - future_destroy(pdata->future); - free(pdata); - runing_over = 1; -} - -void del_future_success(future_result_t* result, void * user) -{ - struct future_pdata *pdata = (struct future_pdata *)user; - - printf("DEL %s succ\n", pdata->filename); - future_destroy(pdata->future); - free(pdata); - runing_over = 1; -} -void del_future_failed(enum e_future_error err, const char * what, void * user) -{ - struct future_pdata *pdata = (struct future_pdata *)user; - - printf("DEL %s fail: %s\n", pdata->filename, what); - future_destroy(pdata->future); - free(pdata); - runing_over = 1; -} - -char * get_file_content(const char *filename, size_t *filelen_out) -{ - char *buffer; - FILE *fp; - size_t filelen = 0; - struct stat filestat; - int readlen; - - fp = fopen(filename, "rb"); - if(fp == NULL) - { - printf("fopen file %s failed.\n", filename); - return NULL; - } - if(fstat(fileno(fp), &filestat)) - { - printf("fstat %s failed.\n", filename); - return NULL; - } - - buffer = (char *)malloc(filestat.st_size); - - while(filelen < (size_t)filestat.st_size) - { - readlen = fread(buffer + filelen, 1, filestat.st_size - filelen, fp); - if(readlen < 0) - { - printf("read error: %s\n", strerror(errno)); - return NULL; - } - - filelen += readlen; - } - fclose(fp); - *filelen_out = filestat.st_size; - - return buffer; -} - -int main(int argc, char **argv) -{ - int n, index=0; - char method[16], filename_in[256], filename_out[256], *p; - struct tango_cache_meta_put putmeta; - struct tango_cache_meta_get getmeta; - struct future_pdata *pdata; - struct cache_evbase_ctx *ctx; - void *runtime_log; - struct tango_cache_parameter *parameter; - - if(argc != 2 && argc!=3) - { - printf("USGAE: %s <PUT/PUTONCE/PUTONCEEV/GET/DEL:filename> [get_out_file_index]\n", argv[0]); - return -1; - } - if(argc==3) - { - index = atoi(argv[2]); - } - - future_promise_library_init(NULL); - - runtime_log = (void *)MESA_create_runtime_log_handle("./runtime.log", 10); - if(NULL==runtime_log) - { - return -1; - } - - cache_evbase_global_init(); - parameter = cache_evbase_parameter_new("./pangu_tg_cahce.conf", "TANGO_CACHE", runtime_log); - assert(parameter != NULL); - instance_asyn = cache_evbase_instance_new(parameter, runtime_log); - assert(instance_asyn!=NULL); - - pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata)); - - if(sscanf(argv[1], "%[^:]:%1023s%n", method, filename_in, &n) != 2) - { - assert(0); - } - if(strlen(filename_in) > 0) - { - memset(&putmeta, 0, sizeof(struct tango_cache_meta_put)); - memset(&getmeta, 0, sizeof(struct tango_cache_meta_get)); - putmeta.url = filename_in; - putmeta.std_hdr[HDR_CONTENT_TYPE] = "Content-Type: maintype/subtype"; - putmeta.std_hdr[HDR_CONTENT_ENCODING] = "Content-Encoding: gzip"; - putmeta.usertag = "Etag: hgdkqkwdwqekdfjwjfjwelkjfkwfejwhf\r\n"; - putmeta.usertag_len = strlen(putmeta.usertag); - - getmeta.url = filename_in; - - p = method; - while(*p=='\r'||*p=='\n') p++; - assert(*p!='\0'); - - if(!strcasecmp(p, "GET")) - { - sprintf(filename_out, "file_index_%u.bin", index); - pdata->future = future_create("_get", get_future_success, get_future_failed, (void *)pdata); - pdata->fp = fopen(filename_out, "w"); - - cache_evbase_fetch_object(instance_asyn, pdata->future, &getmeta, OBJECT_IN_UNKNOWN); - } - else if(!strcasecmp(p, "HEAD")) - { - pdata->future = future_create("_head", head_future_success, head_future_failed, (void *)pdata); - cache_evbase_head_object(instance_asyn, pdata->future, &getmeta); - } - else if(!strcasecmp(p, "DEL")) - { - pdata->future = future_create("_del", del_future_success, del_future_failed, (void *)pdata); - sprintf(pdata->filename, "%s", filename_in); - cache_evbase_delete_object(instance_asyn, pdata->future, filename_in); - } - else if(!strcasecmp(p, "PUTONCE")) - { - size_t filelen; - p = get_file_content(filename_in, &filelen); - pdata->future = future_create("_putonce", put_future_success, put_future_failed, pdata); - - cache_evbase_upload_once_data(instance_asyn, pdata->future, PUT_MEM_FREE, p, filelen, &putmeta, pdata->filename, 256); - } - else if(!strcasecmp(p, "PUTONCEEV")) - { - size_t readlen; - pdata->future = future_create("_putonceev", put_future_success, put_future_failed, (void *)pdata); - struct evbuffer *evbuf = evbuffer_new(); - char buffer[1024]; - - FILE *fp = fopen(filename_in, "rb"); - while(!feof(fp)) - { - readlen = fread(buffer, 1, 1024, fp); - if(readlen < 0) - { - assert(0); - } - evbuffer_add(evbuf, buffer, readlen); - } - fclose(fp); - cache_evbase_upload_once_evbuf(instance_asyn, pdata->future, evbuf, &putmeta, pdata->filename, 256); - } - else - { - pdata->future = future_create("_default", put_future_success, put_future_failed, (void *)pdata); - - ctx = cache_evbase_update_start(instance_asyn, pdata->future, &putmeta); - char buffer[1024]; - FILE *fp = fopen(filename_in, "r"); - while(!feof(fp)) - { - n = fread(buffer, 1, 1024, fp); - assert(n>=0); - cache_evbase_update_frag_data(ctx, PUT_MEM_COPY, buffer, n); - } - - cache_evbase_update_end(ctx, pdata->filename, 256); - } - } - - printf("Waiting to finish.......\n"); - //static int num=0; - while(!runing_over) - { - /*if(++num==10) - { - cache_evbase_update_end(ctx); - }*/ - sleep(1); - } - if(runing_over==2) //GETʱ��ʱɾ���������� - { - sleep(5); - } - - struct cache_statistics out; - cache_evbase_get_statistics(instance_asyn, &out); - printf("-------------------------------------------------------------------------------------------\n" - "get_recv: %llu, get_http: %llu, get_redis: %llu, get_fail_http: %llu, get_fail_redis: %llu, get_miss: %llu\n" - "put_recv: %llu, put_http: %llu, put_redis: %llu, put_fail_http: %llu, put_fail_redis: %llu\n" - "del_recv: %llu, del_succ: %llu, del_fail: %llu, drop_num: %llu, session_redis: %llu, session_http: %llu, memory: %llu\n", - out.get_recv_num, out.get_succ_http, out.get_succ_redis, out.get_err_http, out.get_err_redis, out.get_miss_num, - out.put_recv_num, out.put_succ_http, out.put_succ_redis, out.put_err_http, out.put_err_redis, - out.del_recv_num, out.del_succ_num, out.del_error_num, out.totaldrop_num, out.session_redis, out.session_http, out.memory_used); - return 0; -} - diff --git a/cache/test/lib/libcjson.a b/cache/test/lib/libcjson.a Binary files differdeleted file mode 100644 index 4bf20ae..0000000 --- a/cache/test/lib/libcjson.a +++ /dev/null diff --git a/cache/test/lib/libcrypto.a b/cache/test/lib/libcrypto.a Binary files differdeleted file mode 100644 index 4aea07e..0000000 --- a/cache/test/lib/libcrypto.a +++ /dev/null diff --git a/cache/test/lib/libcurl.a b/cache/test/lib/libcurl.a Binary files differdeleted file mode 100644 index 88b1b70..0000000 --- a/cache/test/lib/libcurl.a +++ /dev/null diff --git a/cache/test/lib/libevent.a b/cache/test/lib/libevent.a Binary files differdeleted file mode 100644 index 772ecb0..0000000 --- a/cache/test/lib/libevent.a +++ /dev/null diff --git a/cache/test/lib/libhiredis_vip.a b/cache/test/lib/libhiredis_vip.a Binary files differdeleted file mode 100644 index a87868b..0000000 --- a/cache/test/lib/libhiredis_vip.a +++ /dev/null diff --git a/cache/test/lib/libssl.a b/cache/test/lib/libssl.a Binary files differdeleted file mode 100644 index ffb498e..0000000 --- a/cache/test/lib/libssl.a +++ /dev/null diff --git a/cache/test/lib/libtango_cache_client.a b/cache/test/lib/libtango_cache_client.a Binary files differdeleted file mode 100644 index e395ce6..0000000 --- a/cache/test/lib/libtango_cache_client.a +++ /dev/null diff --git a/cache/test/lib/libxml2.a b/cache/test/lib/libxml2.a Binary files differdeleted file mode 100644 index 28fea0d..0000000 --- a/cache/test/lib/libxml2.a +++ /dev/null diff --git a/cache/test/pangu_tg_cahce.conf b/cache/test/pangu_tg_cahce.conf deleted file mode 100644 index 1c6d7a2..0000000 --- a/cache/test/pangu_tg_cahce.conf +++ /dev/null @@ -1,46 +0,0 @@ -[TANGO_CACHE] -Addresses of hos, Bucket name in minio. Format is defined by WiredLB. -cache_ip_list=192.168.40.223; -cache_listen_port=9098 -cache_bucket_name=hos/proxy_test - -#cache_ip_list=192.168.44.10; -#cache_listen_port=9090 -#cache_bucket_name=proxybucket - -#Maximum number of connections opened by per host. -#max_connection_per_host=1 -#Maximum number of requests in a pipeline. -#max_cnnt_pipeline_num=20 -#Maximum parellel sessions(http and redis) is allowed to open. -#max_curl_session_num=100 -#Maximum time the request is allowed to take(seconds). -#max_curl_transfer_timeout_s=0 - -#Maximum size of memory used by tango_cache_client. Upload will fail if the current size of memory used exceeds this value. -max_used_memory_size_mb=5120 -#Default TTL of objects, i.e. the time after which the object will expire(minumun 60s, i.e. 1 minute). -cache_default_ttl_second=3600 -#Whether to hash the object key before cache actions. GET/PUT may be faster if you open it. -cache_object_key_hash_switch=1 - -#Store way: 0-HOS; 1-META in REDIS, object in hos; 2-META and small object in Redis, large object in hos; -cache_store_object_way=0 -#If cache_store_object_way is 2 and the size of a object is not bigger than this value, object will be stored in redis. -redis_cache_object_size=20480 -#If cache_store_object_way is not 0, we will use redis to store meta and object. -redis_cluster_ip_list=10.4.35.33-34; -redis_cluster_port_range=9001-9016; -#Configs of WiredLB for Minios load balancer. -#wiredlb_override=1 -#wiredlb_topic= -#wiredlb_datacenter= -wiredlb_health_port=52102 -#wiredlb_group= - -log_fsstat_appname=TANGO_CACHE -log_fsstat_filepath=./field_stat.log -log_fsstat_interval=10 -log_fsstat_trig=1 -log_fsstat_dst_ip=127.0.0.1 -log_fsstat_dst_port=8125 diff --git a/cache/test/tango_cache_test.cpp b/cache/test/tango_cache_test.cpp deleted file mode 100644 index b29fa9d..0000000 --- a/cache/test/tango_cache_test.cpp +++ /dev/null @@ -1,445 +0,0 @@ -#include <event2/event.h> -#include <event2/bufferevent.h> -#include <event.h> -#include <event2/event.h> -#include <event2/buffer.h> -#include <event2/http.h> -#include <event2/http_struct.h> -#include <event2/keyvalq_struct.h> - -#include <sys/ioctl.h> -#include <sys/socket.h> -#include <netinet/in.h> -#include <arpa/inet.h> -#include <net/if.h> -#include <unistd.h> -#include <stdio.h> -#include <stdlib.h> -#include <assert.h> -#include <errno.h> -#include <sys/prctl.h> -#include <sys/un.h> -#include <unistd.h> -#include <sys/poll.h> -#include <curl/curl.h> -#include <event2/event.h> -#include <event2/event_struct.h> -#include <fcntl.h> -#include <sys/stat.h> -#include <errno.h> -#include <sys/cdefs.h> -#include <MESA/MESA_handle_logger.h> - -#include "tango_cache_client.h" - -struct event_base *ev_base; -struct tango_cache_instance *tango_instance; - -#define MSG_OUT stdout /* Send info to stdout, change to stderr if you want */ - -struct future_pdata -{ - struct future * future; - FILE *fp; - char filename[256]; -}; - -void get_future_success(future_result_t* result, void * user) -{ - struct tango_cache_result *res = tango_cache_read_result(result); - struct future_pdata *pdata = (struct future_pdata *)user; - char buffer[1024]; - - switch(res->type) - { - case RESULT_TYPE_USERTAG: - case RESULT_TYPE_HEADER: - memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size); - buffer[res->size] = '\0'; - printf("%s", buffer); - break; - case RESULT_TYPE_BODY: - fwrite(res->data_frag, res->size, 1, pdata->fp); - break; - case RESULT_TYPE_MISS: - printf("cache not hit/fresh\n"); - case RESULT_TYPE_END: - if(res->type != RESULT_TYPE_MISS) - printf("get cache over, total length: %ld\n", res->tlength); - future_destroy(pdata->future); - fclose(pdata->fp); - free(pdata); - break; - default:break; - } -} - -void get_future_failed(enum e_future_error err, const char * what, void * user) -{ - struct future_pdata *pdata = (struct future_pdata *)user; - future_destroy(pdata->future); - fclose(pdata->fp); - free(pdata); - printf("GET fail: %s\n", what); -} - -void head_future_success(future_result_t* result, void * user) -{ - struct tango_cache_result *res = tango_cache_read_result(result); - struct future_pdata *pdata = (struct future_pdata *)user; - char buffer[1024]; - - switch(res->type) - { - case RESULT_TYPE_USERTAG: - case RESULT_TYPE_HEADER: - memcpy(buffer, res->data_frag, res->size>=1024?1023:res->size); - buffer[res->size] = '\0'; - printf("%s", buffer); - break; - case RESULT_TYPE_BODY: - assert(0); - break; - case RESULT_TYPE_MISS: - printf("cache not hit/fresh\n"); - case RESULT_TYPE_END: - if(res->type != RESULT_TYPE_MISS) - printf("HEAD cache over, total length: %ld\n", res->tlength); - future_destroy(pdata->future); - free(pdata); - break; - default:break; - } -} - -void head_future_failed(enum e_future_error err, const char * what, void * user) -{ - struct future_pdata *pdata = (struct future_pdata *)user; - - future_destroy(pdata->future); - free(pdata); - printf("HEAD fail: %s\n", what); -} - -void put_future_success(future_result_t* result, void * user) -{ - struct future_pdata *pdata = (struct future_pdata *)user; - - printf("PUT %s succ\n", pdata->filename); - future_destroy(pdata->future); - free(pdata); -} -void put_future_failed(enum e_future_error err, const char * what, void * user) -{ - struct future_pdata *pdata = (struct future_pdata *)user; - - printf("PUT %s fail: %s\n", what, pdata->filename); - future_destroy(pdata->future); - free(pdata); -} - -void del_future_success(future_result_t* result, void * user) -{ - struct future_pdata *pdata = (struct future_pdata *)user; - - printf("DEL %s succ\n", pdata->filename); - future_destroy(pdata->future); - free(pdata); -} -void del_future_failed(enum e_future_error err, const char * what, void * user) -{ - struct future_pdata *pdata = (struct future_pdata *)user; - - printf("DEL %s fail: %s\n", pdata->filename, what); - future_destroy(pdata->future); - free(pdata); -} - -char * get_file_content(const char *filename, size_t *filelen_out) -{ - char *buffer; - FILE *fp; - size_t filelen = 0; - struct stat filestat; - int readlen; - - fp = fopen(filename, "rb"); - if(fp == NULL) - { - printf("fopen file %s failed.\n", filename); - return NULL; - } - if(fstat(fileno(fp), &filestat)) - { - printf("fstat %s failed.\n", filename); - return NULL; - } - - buffer = (char *)malloc(filestat.st_size); - - while(filelen < (size_t)filestat.st_size) - { - readlen = fread(buffer + filelen, 1, filestat.st_size - filelen, fp); - if(readlen < 0) - { - printf("read error: %s\n", strerror(errno)); - return NULL; - } - - filelen += readlen; - } - fclose(fp); - *filelen_out = filestat.st_size; - - return buffer; -} - -int tango_cache_multi_delete(struct tango_cache_instance *instance, struct future* future, char *objlist[], u_int32_t num); - -static void dummy_accept_callback(evutil_socket_t fd, short events, void *arg) -{ - char s[1024]; - long int rv = 0; - int n = 0; - FILE *input = (FILE *)arg; - static int index=0; - char filename[128], method[16], buffer[1024], *p; - char *dellist[16]; - char *pstart, *save_ptr=NULL; - int delnum=0; - struct tango_cache_meta_put putmeta; - struct tango_cache_meta_get getmeta; - struct future_pdata *pdata; - - struct tango_cache_ctx *ctx; - - do - { - s[0]='\0'; - rv = fscanf(input, "%[^:]:%1023s%n", method, s, &n); - if(strlen(s) > 0) - { - p = method; - - memset(&putmeta, 0, sizeof(struct tango_cache_meta_put)); - memset(&getmeta, 0, sizeof(struct tango_cache_meta_get)); - putmeta.url = s; - putmeta.std_hdr[HDR_CONTENT_TYPE] = "Content-Type: maintype/subtype"; - putmeta.std_hdr[HDR_CONTENT_ENCODING] = "Content-Encoding: gzip"; - putmeta.usertag = "Etag: hgdkqkwdwqekdfjwjfjwelkjfkwfejwhf\r\n"; - putmeta.usertag_len = strlen(putmeta.usertag); - - getmeta.url = s; - - while(*p=='\r'||*p=='\n')p++; - if(*p=='\0') continue; - if(!strcasecmp(p, "GET")) - { - sprintf(filename, "file_index_%u.bin", index++); - pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata)); - pdata->fp = fopen(filename, "w"); - pdata->future = future_create("_get", get_future_success, get_future_failed, (void *)pdata); - - if(tango_cache_fetch_object(tango_instance, pdata->future, &getmeta, OBJECT_IN_UNKNOWN) < 0) - { - get_future_failed(FUTURE_ERROR_CANCEL, "", pdata); - } - } - else if(!strcasecmp(p, "HEAD")) - { - pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata)); - pdata->future = future_create("_head", head_future_success, head_future_failed, (void *)pdata); - - if(tango_cache_head_object(tango_instance, pdata->future, &getmeta) < 0) - { - head_future_failed(FUTURE_ERROR_CANCEL, "", pdata); - } - } - else if(!strcasecmp(p, "PUTONCE")) - { - size_t filelen; - p = get_file_content(s, &filelen); - pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata)); - pdata->future = future_create("_putnoce", put_future_success, put_future_failed, (void *)pdata); - - if(tango_cache_upload_once_data(tango_instance, pdata->future, PUT_MEM_FREE, p, filelen, &putmeta, pdata->filename, 256)) - { - put_future_failed(FUTURE_ERROR_CANCEL, "", pdata); - } - } - else if(!strcasecmp(p, "PUTONCEEV")) - { - size_t readlen; - pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata)); - pdata->future = future_create("_putonceev", put_future_success, put_future_failed, pdata); - struct evbuffer *evbuf = evbuffer_new(); - char buffer[1024]; - - FILE *fp = fopen(s, "rb"); - while(!feof(fp)) - { - readlen = fread(buffer, 1, 1024, fp); - if(readlen < 0) - { - assert(0); - } - evbuffer_add(evbuf, buffer, readlen); - } - fclose(fp); - if(tango_cache_upload_once_evbuf(tango_instance, pdata->future, EVBUFFER_MOVE, evbuf, &putmeta, pdata->filename, 256)) - { - put_future_failed(FUTURE_ERROR_CANCEL, "", pdata); - } - } - else if(!strcasecmp(p, "DEL")) - { - pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata)); - pdata->future = future_create("_del", del_future_success, del_future_failed, pdata); - sprintf(pdata->filename, "%s", s); - tango_cache_delete_object(tango_instance, pdata->future, s); - } - else if(!strcasecmp(p, "DELMUL")) //TODO - { - pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata)); - pdata->future = future_create("_delmul", del_future_success, del_future_failed, pdata); - sprintf(pdata->filename, "%s", s); - - for(pstart = strtok_r(s, ";", &save_ptr); pstart != NULL; pstart = strtok_r(NULL, ";", &save_ptr)) - { - dellist[delnum++] = pstart; - } - tango_cache_multi_delete(tango_instance, pdata->future, dellist, delnum); - } - else - { - pdata = (struct future_pdata *)malloc(sizeof(struct future_pdata)); - pdata->future = future_create("_default", put_future_success, put_future_failed, pdata); - - ctx = tango_cache_update_start(tango_instance, pdata->future, &putmeta); - if(ctx==NULL) - { - put_future_failed(FUTURE_ERROR_CANCEL, "tango_cache_update_start_NULL", pdata); - continue; - } - FILE *fp = fopen(s, "r"); - while(!feof(fp)) - { - n = fread(buffer, 1, 1024, fp); - assert(n>=0); - tango_cache_update_frag_data(ctx, buffer, n); - } - fclose(fp); - if(tango_cache_update_end(ctx, pdata->filename, 256) < 0) - { - put_future_failed(FUTURE_ERROR_CANCEL, "", pdata); - } - } - } - else - { - break; - } - } while(rv != EOF); -} - -struct event fifo_event; -static int init_fifo(void) -{ - struct stat st; - curl_socket_t sockfd; - static const char *fifo = "cache.fifo"; - - fprintf(MSG_OUT, "Creating named pipe \"%s\"\n", fifo); - if(lstat (fifo, &st) == 0) { - if((st.st_mode & S_IFMT) == S_IFREG) { - errno = EEXIST; - perror("lstat"); - exit(1); - } - } - unlink(fifo); - if(mkfifo (fifo, 0600) == -1) { - perror("mkfifo"); - exit(1); - } - sockfd = open(fifo, O_RDWR | O_NONBLOCK, 0); - if(sockfd == -1) { - perror("open"); - exit(1); - } - FILE *input = fdopen(sockfd, "r"); - - memset(&fifo_event, 0, sizeof(struct event)); - fprintf(MSG_OUT, "Now, pipe some URL's into > %s\n", fifo); - event_assign(&fifo_event, ev_base, sockfd, EV_READ|EV_PERSIST, - dummy_accept_callback, input); - event_add(&fifo_event, NULL); - return (0); -} - -void timer_cb(evutil_socket_t fd, short what, void *arg) -{ - struct timeval tv; - struct cache_statistics out; - - tv.tv_sec = 10; - tv.tv_usec = 0; - - /*static int num=0; - num++; - if(ctx_global!=NULL && num>5) - { - tango_cache_update_end(ctx_global); - ctx_global = NULL; - }*/ - - tango_cache_get_statistics(tango_instance, &out); - printf("-------------------------------------------------------------------------------------------\n" - "get_recv: %llu, get_http: %llu, get_redis: %llu, get_fail_http: %llu, get_fail_redis: %llu, get_miss: %llu\n" - "put_recv: %llu, put_http: %llu, put_redis: %llu, put_fail_http: %llu, put_fail_redis: %llu\n" - "del_recv: %llu, del_succ: %llu, del_fail: %llu, drop_num: %llu, session_redis: %llu, session_http: %llu, memory: %llu\n", - out.get_recv_num, out.get_succ_http, out.get_succ_redis, out.get_err_http, out.get_err_redis, out.get_miss_num, - out.put_recv_num, out.put_succ_http, out.put_succ_redis, out.put_err_http, out.put_err_redis, - out.del_recv_num, out.del_succ_num, out.del_error_num, out.totaldrop_num, out.session_redis, out.session_http, out.memory_used); - - event_add((struct event *)arg, &tv); -} - -int main(int crgc, char **arg) -{ - struct event ev_timer; - struct timeval tv; - void *runtime_log; - struct tango_cache_parameter *parameter; - - future_promise_library_init(NULL); - - runtime_log = (void *)MESA_create_runtime_log_handle("./runtime.log", 10); - if(NULL==runtime_log) - { - return -1; - } - - ev_base = event_base_new(); - if (!ev_base) - { - printf("create socket error!\n"); - return 0; - } - init_fifo(); - - tango_cache_global_init(); - parameter = tango_cache_parameter_new("./pangu_tg_cahce.conf", "TANGO_CACHE", runtime_log); - assert(parameter != NULL); - tango_instance = tango_cache_instance_new(parameter, ev_base, runtime_log); - - tv.tv_sec = 10; - tv.tv_usec = 0; - evtimer_assign(&ev_timer, ev_base, timer_cb, &ev_timer); - evtimer_add(&ev_timer, &tv); - - event_base_dispatch(ev_base); - return 0; -} - - diff --git a/cache/test/test_cache_pending.cpp b/cache/test/test_cache_pending.cpp deleted file mode 100644 index b21c335..0000000 --- a/cache/test/test_cache_pending.cpp +++ /dev/null @@ -1,211 +0,0 @@ -#include <iostream> -#include <gtest/gtest.h> -#include <tango_cache_pending.h> - -using namespace std; - -int main(int argc, char **argv) -{ - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -TEST(CacheActionTest, PragmaField) -{ - struct tfe_http_field http_heads; - struct request_freshness restrict; - struct response_freshness freshness; - - http_heads.http_field = TFE_HTTP_PRAGMA; - http_heads.value = "no-cache"; - EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1,&restrict), REVALIDATE); - EXPECT_EQ(restrict.min_fresh, 0); - EXPECT_EQ(restrict.max_age, 0); - EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), REVALIDATE); - EXPECT_EQ(freshness.date, 0); - EXPECT_EQ(freshness.last_modified, 0); - EXPECT_EQ(freshness.timeout, 0); - -} - -TEST(CacheActionTest, CacheCtlNoCache) -{ - struct tfe_http_field http_heads; - struct request_freshness restrict; - struct response_freshness freshness; - http_heads.http_field = TFE_HTTP_CACHE_CONTROL; - http_heads.value = "no-cache"; - EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1, &restrict), REVALIDATE); - EXPECT_EQ(restrict.min_fresh, 0); - EXPECT_EQ(restrict.max_age, 0); - EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), REVALIDATE); - EXPECT_EQ(freshness.date, 0); - EXPECT_EQ(freshness.last_modified, 0); - EXPECT_EQ(freshness.timeout, 0); -} - -TEST(CacheActionTest, CacheCtlNoStore) -{ - struct tfe_http_field http_heads; - struct request_freshness restrict; - struct response_freshness freshness; - http_heads.http_field = TFE_HTTP_CACHE_CONTROL; - http_heads.value = "no-store"; - EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1, &restrict), FORBIDDEN); - EXPECT_EQ(restrict.min_fresh, 0); - EXPECT_EQ(restrict.max_age, 0); - EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), FORBIDDEN); - EXPECT_EQ(freshness.date, 0); - EXPECT_EQ(freshness.last_modified, 0); - EXPECT_EQ(freshness.timeout, 0); -} -TEST(CacheActionTest, CacheCtlCached) -{ - struct tfe_http_field http_heads; - struct request_freshness restrict; - struct response_freshness freshness; - http_heads.http_field = TFE_HTTP_CACHE_CONTROL; - http_heads.value = "only-if-cached, max-age=3600"; - EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1, &restrict), ALLOWED); - EXPECT_EQ(restrict.min_fresh, 0); - EXPECT_EQ(restrict.max_age, 3600); - EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), ALLOWED); - EXPECT_EQ(freshness.date, 0); - EXPECT_EQ(freshness.last_modified, 0); - EXPECT_EQ(freshness.timeout, 3600); -} - -TEST(CacheActionTest, CacheCtlMinFresh) -{ - struct tfe_http_field http_heads; - struct request_freshness restrict; - http_heads.http_field = TFE_HTTP_CACHE_CONTROL; - http_heads.value = "only-if-cached, max-age=3600, min-fresh=60"; - EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1, &restrict), ALLOWED); - EXPECT_EQ(restrict.min_fresh, 60); - EXPECT_EQ(restrict.max_age, 3600); -} - -TEST(CacheActionTest, CacheCtlMustRevalidate) -{ - struct tfe_http_field http_heads; - struct response_freshness freshness; - http_heads.http_field = TFE_HTTP_CACHE_CONTROL; - http_heads.value = "must-revalidate"; - EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), REVALIDATE); - EXPECT_EQ(freshness.date, 0); - EXPECT_EQ(freshness.last_modified, 0); - EXPECT_EQ(freshness.timeout, 0); -} -TEST(CacheActionTest, CacheCtlProxyRevalidate) -{ - struct tfe_http_field http_heads; - struct response_freshness freshness; - http_heads.http_field = TFE_HTTP_CACHE_CONTROL; - http_heads.value = "proxy-revalidate"; - EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), REVALIDATE); - EXPECT_EQ(freshness.date, 0); - EXPECT_EQ(freshness.last_modified, 0); - EXPECT_EQ(freshness.timeout, 0); -} -TEST(CacheActionTest, CacheCtlCachedSMaxAge) -{ - struct tfe_http_field http_heads; - struct response_freshness freshness; - http_heads.http_field = TFE_HTTP_CACHE_CONTROL; - http_heads.value = "public, max-age=60, s-maxage=3600"; - EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), ALLOWED); - EXPECT_EQ(freshness.date, 0); - EXPECT_EQ(freshness.last_modified, 0); - EXPECT_EQ(freshness.timeout, 3600); -} - - -TEST(CacheActionTest, CacheCtlPrivate) -{ - struct tfe_http_field http_heads; - struct response_freshness freshness; - http_heads.http_field = TFE_HTTP_CACHE_CONTROL; - http_heads.value = "private"; - EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), FORBIDDEN); - EXPECT_EQ(freshness.date, 0); - EXPECT_EQ(freshness.last_modified, 0); - EXPECT_EQ(freshness.timeout, 0); -} -TEST(CacheActionTest, ExpiresResponse) -{ - struct tfe_http_field http_heads; - struct response_freshness freshness; - http_heads.http_field = TFE_HTTP_EXPIRES; - http_heads.value = "Sun, 01 Dec 2019 16:00:00 GMT"; - EXPECT_EQ(tfe_cache_put_pending(&http_heads, 1, &freshness), ALLOWED); - EXPECT_EQ(freshness.date, 0); - EXPECT_EQ(freshness.last_modified, 0); - cout << freshness.timeout << endl; -} -TEST(CacheActionTest, IfMatchRequest) -{ - struct tfe_http_field http_heads; - struct request_freshness restrict; - http_heads.http_field = TLF_HTTP_IF_MATCH; - http_heads.value = "50b1c1d4f775c61:df3"; - EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1, &restrict), REVALIDATE); - EXPECT_EQ(restrict.min_fresh, 0); - EXPECT_EQ(restrict.max_age, 0); -} -TEST(CacheActionTest, IfNoMatchRequest) -{ - struct tfe_http_field http_heads; - struct request_freshness restrict; - http_heads.http_field = TLF_HTTP_IF_NONE_MATCH; - http_heads.value = "50b1c1d4f775c61:df3"; - EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1, &restrict), REVALIDATE); - EXPECT_EQ(restrict.min_fresh, 0); - EXPECT_EQ(restrict.max_age, 0); -} -TEST(CacheActionTest, IfModifiedSinceRequest) -{ - struct tfe_http_field http_heads; - struct request_freshness restrict; - http_heads.http_field = TLF_HTTP_IF_MODIFIED_SINCE; - http_heads.value = "Sun, 01 Dec 2019 16:00:00 GMT"; - EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1, &restrict), REVALIDATE); - EXPECT_EQ(restrict.min_fresh, 0); - EXPECT_EQ(restrict.max_age, 0); -} -TEST(CacheActionTest, IfUnModifiedSinceRequest) -{ - struct tfe_http_field http_heads; - struct request_freshness restrict; - http_heads.http_field = TLF_HTTP_IF_UNMODIFIED_SINCE; - http_heads.value = "Sun, 01 Dec 2019 16:00:00 GMT"; - EXPECT_EQ(tfe_cache_get_pending(&http_heads, 1, &restrict), REVALIDATE); - EXPECT_EQ(restrict.min_fresh, 0); - EXPECT_EQ(restrict.max_age, 0); -} -TEST(CacheActionTest, Date) -{ - struct tfe_http_field http_heads[2]; - struct response_freshness freshness; - http_heads[0].http_field=TFE_HTTP_CACHE_CONTROL; - http_heads[0].value = "public, max=3600"; - http_heads[1].http_field = TFE_HTTP_DATE; - http_heads[1].value = "Sun, 01 Dec 2019 16:00:00 GMT"; - EXPECT_EQ(tfe_cache_put_pending(http_heads, 2, &freshness), ALLOWED); - cout << freshness.date << endl; - EXPECT_EQ(freshness.last_modified, 0); - EXPECT_EQ(freshness.timeout, 0); -} -TEST(CacheActionTest, LastModified) -{ - struct tfe_http_field http_heads[2]; - struct response_freshness freshness; - http_heads[0].http_field=TFE_HTTP_CACHE_CONTROL; - http_heads[0].value = "public, max=3600"; - http_heads[1].http_field = TLF_HTTP_LAST_MODIFIED; - http_heads[1].value = "Sun, 01 Dec 2019 16:00:00 GMT"; - EXPECT_EQ(tfe_cache_put_pending(http_heads, 2, &freshness),ALLOWED); - cout << freshness.last_modified << endl; - EXPECT_EQ(freshness.date, 0); - EXPECT_EQ(freshness.timeout, 0); -} diff --git a/cache/test/urllist_get.txt b/cache/test/urllist_get.txt deleted file mode 100644 index 1112d09..0000000 --- a/cache/test/urllist_get.txt +++ /dev/null @@ -1,3 +0,0 @@ -GET:openbucket/curl_put_chunk_1.pcap -GET:openbucket/tcpdump_all_1.pcap -GET:openbucket/tcpdump_big_1.pcap diff --git a/cache/test/urllist_put.txt b/cache/test/urllist_put.txt deleted file mode 100644 index 2522f9c..0000000 --- a/cache/test/urllist_put.txt +++ /dev/null @@ -1,3 +0,0 @@ -PUT:openbucket/curl_put_chunk_1.pcap -PUT:openbucket/tcpdump_all_1.pcap -PUT:openbucket/tcpdump_big_1.pcap diff --git a/cmake/Package.cmake b/cmake/Package.cmake index 65c5c96..3ea7b23 100644 --- a/cmake/Package.cmake +++ b/cmake/Package.cmake @@ -31,8 +31,7 @@ else() endif() # setup %config(noreplace) -set(CPACK_RPM_USER_FILELIST "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/pangu/pangu_pxy.conf" - "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/tfe/tfe.conf" +set(CPACK_RPM_USER_FILELIST "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/tfe/tfe.conf" "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/tfe/zlog.conf" "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/tfe/future.conf" "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/doh/doh.conf") diff --git a/conf/CMakeLists.txt b/conf/CMakeLists.txt index 4267a06..48c9565 100644 --- a/conf/CMakeLists.txt +++ b/conf/CMakeLists.txt @@ -1,3 +1,2 @@ install(DIRECTORY tfe DESTINATION conf COMPONENT Profile) -install(DIRECTORY pangu DESTINATION conf COMPONENT Profile) install(DIRECTORY doh DESTINATION conf COMPONENT Profile)
\ No newline at end of file diff --git a/conf/doh/doh.conf b/conf/doh/doh.conf index dd78fdb..2bf4ea0 100644 --- a/conf/doh/doh.conf +++ b/conf/doh/doh.conf @@ -6,8 +6,8 @@ table_appid=ATTR_APP_ID table_src_addr=ATTR_SOURCE_ADDR table_dst_addr=ATTR_DESTINATION_ADDR table_qname=ATTR_DOH_QNAME -table_host=ATTR_DOH_HOST -table_host_cat=ATTR_DOH_HOST_CAT +table_host=ATTR_SERVER_FQDN +table_host_cat=ATTR_SERVER_FQDN_CAT [kafka] ENTRANCE_ID=0 diff --git a/conf/tfe/tfe.conf b/conf/tfe/tfe.conf index c80f87b..b9e50b2 100644 --- a/conf/tfe/tfe.conf +++ b/conf/tfe/tfe.conf @@ -116,6 +116,10 @@ cert_expire_time=24 # health_check only for "mode=normal" default 1 enable_health_check=1 +[tsg_http] +enable_plugin=1 +en_sendlog=1 + [debug] # 1 : enforce tcp passthrough # 0 : Whether to passthrough depends on the tcp_options in cmsg diff --git a/platform/CMakeLists.txt b/platform/CMakeLists.txt index d0612ff..b2fe0a6 100644 --- a/platform/CMakeLists.txt +++ b/platform/CMakeLists.txt @@ -10,7 +10,7 @@ target_include_directories(tfe PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/external target_include_directories(tfe PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include/internal) target_include_directories(tfe PRIVATE ${SYSTEMD_INCLUDE_DIRS}) -target_link_libraries(tfe common tango-cache-client) +target_link_libraries(tfe common) target_link_libraries(tfe pthread dl nfnetlink libnetfilter_queue-static openssl-ssl-static diff --git a/platform/src/proxy.cpp b/platform/src/proxy.cpp index 32117c8..11d5cb4 100644 --- a/platform/src/proxy.cpp +++ b/platform/src/proxy.cpp @@ -32,7 +32,6 @@ #include <MESA/MESA_prof_load.h> #include <MESA/field_stat2.h> -#include <tango_cache_client.h> #include <tfe_utils.h> #include <tfe_future.h> #include <tfe_stream.h> @@ -651,8 +650,6 @@ int main(int argc, char * argv[]) FS_library_init(); future_promise_library_init(future_profile); - tango_cache_global_init(); - /* CONFIG */ ret = tfe_proxy_config(g_default_proxy, main_profile); CHECK_OR_EXIT(ret == 0, "Failed at loading profile %s, Exit.", main_profile); diff --git a/plugin/business/doh/src/doh.cpp b/plugin/business/doh/src/doh.cpp index c5e080a..f8f854a 100644 --- a/plugin/business/doh/src/doh.cpp +++ b/plugin/business/doh/src/doh.cpp @@ -386,8 +386,8 @@ static int doh_maat_init(const char *profile, const char *section) MESA_load_profile_string_def(profile, section, "table_src_addr", g_doh_conf->tables[TYPE_SRC_ADDR].name, TFE_STRING_MAX, "ATTR_SOURCE_ADDR"); MESA_load_profile_string_def(profile, section, "table_dst_addr", g_doh_conf->tables[TYPE_DST_ADDR].name, TFE_STRING_MAX, "ATTR_DESTINATION_ADDR"); MESA_load_profile_string_def(profile, section, "table_qname", g_doh_conf->tables[TYPE_QNAME].name, TFE_STRING_MAX, "ATTR_DOH_QNAME"); - MESA_load_profile_string_def(profile, section, "table_host", g_doh_conf->tables[TYPE_HOST].name, TFE_STRING_MAX, "ATTR_DOH_HOST"); - MESA_load_profile_string_def(profile, section, "table_host_cat", g_doh_conf->tables[TYPE_HOST_CAT].name, TFE_STRING_MAX, "ATTR_DOH_HOST_CAT"); + MESA_load_profile_string_def(profile, section, "table_host", g_doh_conf->tables[TYPE_HOST].name, TFE_STRING_MAX, "ATTR_SERVER_FQDN"); + MESA_load_profile_string_def(profile, section, "table_host_cat", g_doh_conf->tables[TYPE_HOST_CAT].name, TFE_STRING_MAX, "ATTR_SERVER_FQDN_CAT"); MESA_load_profile_string_def(profile, section, "table_internal_addr", g_doh_conf->tables[TYPE_INTERNAL_ADDR].name, TFE_STRING_MAX, "ATTR_INTERNAL_ADDR"); MESA_load_profile_string_def(profile, section, "table_external_addr", g_doh_conf->tables[TYPE_EXTERNAL_ADDR].name, TFE_STRING_MAX, "ATTR_EXTERNAL_ADDR"); diff --git a/plugin/business/tsg-http/CMakeLists.txt b/plugin/business/tsg-http/CMakeLists.txt index 9c5fc9a..c12b2f2 100644 --- a/plugin/business/tsg-http/CMakeLists.txt +++ b/plugin/business/tsg-http/CMakeLists.txt @@ -1,7 +1,7 @@ -add_library(tsg-http src/tsg_logger.cpp src/tsg_http.cpp src/pattern_replace.cpp src/tsg_web_cache.cpp src/edit_element.cpp src/http_lua.cpp) +add_library(tsg-http src/tsg_logger.cpp src/tsg_http.cpp src/pattern_replace.cpp src/edit_element.cpp src/http_lua.cpp) target_include_directories(tsg-http PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include) -target_link_libraries(tsg-http PUBLIC common http tango-cache-client) -target_link_libraries(tsg-http PUBLIC rdkafka ctemplate-static cjson pcre2-static ratelimiter-static libdablooms pthread) +target_link_libraries(tsg-http PUBLIC common http) +target_link_libraries(tsg-http PUBLIC rdkafka ctemplate-static cjson pcre2-static libdablooms pthread) target_link_libraries(tsg-http PUBLIC maatframe tsglua) target_link_libraries(tsg-http PUBLIC libxml2-static z) diff --git a/plugin/business/tsg-http/include/tsg_proxy_web_cache.h b/plugin/business/tsg-http/include/tsg_proxy_web_cache.h deleted file mode 100644 index 4ebfe23..0000000 --- a/plugin/business/tsg-http/include/tsg_proxy_web_cache.h +++ /dev/null @@ -1,69 +0,0 @@ -#pragma once -#include <event2/event.h> -#include <event2/buffer.h> -#include <tfe_future.h> -#include <MESA/maat.h> - - -struct cache_handle; -struct cache_handle* create_web_cache_handle(const char* profile_path, const char* section, - struct event_base* gc_evbase, struct maat *feather, void *logger); - -struct cached_meta -{ - size_t content_length; - char* content_type; - char* last_modified; - char* etag; -}; -const struct cached_meta* cache_query_result_read_meta(future_result_t * result); -size_t cache_query_result_get_data(future_result_t * result, const unsigned char** pp_data); - -int web_cache_async_read(struct cache_handle* handle, unsigned int thread_id, - const struct tfe_http_half * request, struct cache_mid** mid, struct future* f); - - - -enum cache_query_result_type -{ - CACHE_QUERY_RESULT_MISS, - CACHE_QUERY_RESULT_META, - CACHE_QUERY_RESULT_DATA, - CACHE_QUERY_RESULT_END, - CACHE_QUERY_RESULT_IRRELEVANT, - __CACHE_QUERY_RESULT_MAX -}; -enum cache_query_result_type cache_query_result_get_type(future_result_t * result); - -enum cache_pending_result -{ - PENDING_RESULT_NONE=0, - PENDING_RESULT_FOBIDDEN, - PENDING_RESULT_REVALIDATE, - PENDING_RESULT_ALLOWED, - PENDING_RESULT_HIT, - PENDING_RESULT_MISS -}; -const char* cache_pending_result_string(enum cache_pending_result); -struct cache_mid; -void cache_mid_clear(struct cache_mid **mid); - -const struct cached_meta* cache_pending_result_read_meta(future_result_t * result, struct cache_mid* mid); -enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, unsigned int thread_id, - const struct tfe_http_half * request, struct cache_mid **mid, struct future* f_revalidate); - - - - -struct cache_write_context; -struct cache_write_context* web_cache_write_start(struct cache_handle* handle, unsigned int thread_id, - const struct tfe_http_session * session, struct cache_mid **mid); -void web_cache_write(struct cache_write_context* ctx, const unsigned char * body_frag, size_t frag_size); -//return 1 on success -//return -1 on failed -//return -2 on cancel - -int web_cache_write_end(struct cache_write_context* ctx); - - - diff --git a/plugin/business/tsg-http/src/tsg_http.cpp b/plugin/business/tsg-http/src/tsg_http.cpp index 5d6b0f9..8e34112 100644 --- a/plugin/business/tsg-http/src/tsg_http.cpp +++ b/plugin/business/tsg-http/src/tsg_http.cpp @@ -2,7 +2,6 @@ #include "edit_element.h" #include "pattern_replace.h" #include "http_lua.h" -#include "tsg_proxy_web_cache.h" #include <tfe_proxy.h> #include <tfe_stream.h> @@ -31,8 +30,6 @@ #include <string.h> #include <sys/types.h> -#include "ratelimiter.h" - #define MAX_EDIT_ZONE_NUM 64 enum proxy_action //Bigger action number is prior. @@ -162,10 +159,6 @@ struct tsg_proxy_rt int plolicy_table_id[POLICY_PROFILE_TABLE_MAX]; ctemplate::Template * tpl_403, * tpl_404, * tpl_451; - long long suspend_max; - int cache_enabled; - struct cache_handle* cache; - screen_stat_handle_t fs_handle; long long stat_val[__PX_STAT_MAX]; int fs_id[__PX_STAT_MAX]; @@ -173,7 +166,6 @@ struct tsg_proxy_rt struct event* gcev; struct tsg_lua_script lua_script; - Ratelimiter_handle_t ratelimiter; int enable_rate; int ctrl_plugin_idx; @@ -1022,8 +1014,8 @@ int proxy_policy_init(const char* profile_path, const char* static_section, cons table_name[PXY_CTRL_SOURCE_ADDR] = "ATTR_SOURCE_ADDR"; table_name[PXY_CTRL_DESTINATION_ADDR]="ATTR_DESTINATION_ADDR"; table_name[PXY_CTRL_HTTP_URL] = "ATTR_HTTP_URL"; - table_name[PXY_CTRL_HTTP_FQDN] = "ATTR_HTTP_HOST_VIRTUAL"; - table_name[PXY_CTRL_HTTP_FQDN_CAT] = "ATTR_HTTP_HOST_CAT_VIRTUAL"; + table_name[PXY_CTRL_HTTP_FQDN] = "ATTR_SERVER_FQDN"; + table_name[PXY_CTRL_HTTP_FQDN_CAT] = "ATTR_SERVER_FQDN_CAT"; table_name[PXY_CTRL_HTTP_REQ_HDR] = "ATTR_HTTP_REQ_HDR"; table_name[PXY_CTRL_HTTP_REQ_BODY] = "ATTR_HTTP_REQ_BODY"; table_name[PXY_CTRL_HTTP_RES_HDR] = "ATTR_HTTP_RES_HDR"; @@ -1084,64 +1076,12 @@ error_out: return ret; } -Ratelimiter_handle_t ratelimit_handle_create(const char* profile_path, const char* static_section) -{ - int ret=0, field_stat=0; - int redis_db_idx = 0; - int redis_port = 0, interval_sec = 0; - char redis_server[TFE_STRING_MAX] = {0}; - char token_name[TFE_STRING_MAX] = {0}; - - Ratelimiter_handle_t ratelimit = NULL; - - MESA_load_profile_int_def(profile_path, static_section, "enable", &(g_proxy_rt->enable_rate), 0); - MESA_load_profile_int_def(profile_path, static_section, "redis_port", &(redis_port), 6379); - MESA_load_profile_string_def(profile_path, static_section, "redis_server", redis_server, sizeof(redis_server), ""); - MESA_load_profile_string_def(profile_path, static_section, "token_name", token_name, sizeof(token_name), ""); - MESA_load_profile_int_def(profile_path, static_section, "redis_db_index", &(redis_db_idx), 0); - - MESA_load_profile_int_def(profile_path, static_section, "interval_sec", &(interval_sec), 1); - if (g_proxy_rt->enable_rate != 1) - { - return NULL; - } - ratelimit=Ratelimiter_create(token_name, g_proxy_rt->local_logger); - Ratelimiter_set_opt(ratelimit, RATELIMITER_OPT_INTERVAL_SEC, &interval_sec, sizeof(interval_sec)); - Ratelimiter_set_opt(ratelimit, RATELIMITER_OPT_REDIS_IP, redis_server, strlen(redis_server) + 1); - Ratelimiter_set_opt(ratelimit, RATELIMITER_OPT_REDIS_PORT, &redis_port, sizeof(redis_port)); - Ratelimiter_set_opt(ratelimit, RATELIMITER_OPT_REDIS_INDEX, &redis_db_idx, sizeof(redis_db_idx)); - - /*field stata for debug test**/ - MESA_load_profile_int_def(profile_path, static_section, "field_stat", &(field_stat), 0); - if (field_stat == 1) - { - MESA_load_profile_int_def(profile_path, static_section, "stat_port", &(redis_port), 6379); - MESA_load_profile_string_def(profile_path, static_section, "stat_server", redis_server, sizeof(redis_server), ""); - Ratelimiter_set_opt(ratelimit, RATELIMITER_OPT_FIELD_STAT, &field_stat, sizeof(field_stat)); - Ratelimiter_set_opt(ratelimit, RATELIMITER_OPT_STAT_IP, redis_server, strlen(redis_server)); - Ratelimiter_set_opt(ratelimit, RATELIMITER_OPT_STAT_PORT, &redis_port, sizeof(redis_port)); - Ratelimiter_set_opt(ratelimit, RATELIMITER_OPT_STAT_PATH, token_name, sizeof(token_name)); - Ratelimiter_stat_init(ratelimit); - } - ret = Ratelimiter_start(ratelimit); - if (ret < 0) - { - TFE_LOG_ERROR(g_proxy_rt->local_logger, "%s Ratelimiter init failed.", __FUNCTION__); - goto error_out; - } - return ratelimit; -error_out: - Ratelimiter_stop(ratelimit); - return NULL; -} - int proxy_http_init(struct tfe_proxy * proxy) { - const char * profile_path = "./conf/pangu/pangu_pxy.conf";; - int temp=0; + const char * profile_path = "./conf/tfe/tfe.conf";; g_proxy_rt = ALLOC(struct tsg_proxy_rt, 1); - MESA_load_profile_int_def(profile_path, "DEBUG", "enable_plugin", &(g_proxy_rt->enable_plugin), 1); + MESA_load_profile_int_def(profile_path, "tsg_http", "enable_plugin", &(g_proxy_rt->enable_plugin), 1); if (!g_proxy_rt->enable_plugin) { return 0; @@ -1152,7 +1092,7 @@ int proxy_http_init(struct tfe_proxy * proxy) g_proxy_rt->local_logger = (void *)MESA_create_runtime_log_handle("tsg_http", RLOG_LV_DEBUG); - g_proxy_rt->send_logger = proxy_log_handle_create(profile_path, "LOG", g_proxy_rt->local_logger); + g_proxy_rt->send_logger = proxy_log_handle_create(profile_path, "tsg_http", g_proxy_rt->local_logger); if (!g_proxy_rt->send_logger) { goto error_out; @@ -1160,7 +1100,6 @@ int proxy_http_init(struct tfe_proxy * proxy) g_proxy_rt->fs_handle = tfe_proxy_get_fs_handle(); proxy_http_stat_init(g_proxy_rt); - g_proxy_rt->ratelimiter=ratelimit_handle_create(profile_path, "ratelimit"); if(http_lua_handle_create(&g_proxy_rt->lua_script, g_proxy_rt->thread_num, "tfe") <0) { goto error_out; @@ -1183,22 +1122,8 @@ int proxy_http_init(struct tfe_proxy * proxy) MESA_load_profile_string_def(profile_path, "TEMPLATE", "PAGE_451", page_path, sizeof(page_path), "./resource/pangu/HTTP451.html"); g_proxy_rt->tpl_451 = ctemplate::Template::GetTemplate(page_path, ctemplate::DO_NOT_STRIP); + g_proxy_rt->enable_rate=0; - MESA_load_profile_int_def(profile_path, "TANGO_CACHE", "suspend_max", &(temp), 1024*1024); - g_proxy_rt->suspend_max=temp; - - MESA_load_profile_int_def(profile_path, "TANGO_CACHE", "enable_cache", &(g_proxy_rt->cache_enabled), 1); - if(g_proxy_rt->cache_enabled) - { - g_proxy_rt->cache = create_web_cache_handle(profile_path, "TANGO_CACHE", g_proxy_rt->gc_evbase, - g_proxy_rt->feather, g_proxy_rt->local_logger); - if(!g_proxy_rt->cache) - { - TFE_LOG_INFO(NULL, "Tango Cache init failed."); - goto error_out; - } - TFE_LOG_INFO(NULL, "Tango Cache Enabled."); - } TFE_LOG_INFO(NULL, "Tsg_pxy HTTP init success."); return 0; @@ -1250,7 +1175,6 @@ struct proxy_http_ctx long long result[MAX_SCAN_RESULT]; struct maat_state *scan_mid; struct maat_stream *sp; - struct cache_mid* cmid; struct maat_rule_t * enforce_rules; size_t n_enforce; struct policy_action_param *param; @@ -1267,15 +1191,7 @@ struct proxy_http_ctx int (* resumed_cb)(const struct tfe_stream * stream, const struct tfe_http_session * session, enum tfe_http_event event, const unsigned char * data, size_t datalen, unsigned int thread_id, struct proxy_http_ctx* ctx); - - enum cache_pending_result pending_result; - struct future *f_cache_pending, *f_cache_query; struct tfe_http_session * ref_session; - struct tfe_http_half* cache_revalidate_req; - struct tfe_http_half* cached_response; - size_t cache_result_declared_sz, cache_result_actual_sz; - struct cache_write_context* cache_write_ctx; - int cache_wirte_result; size_t c2s_byte_num; size_t s2c_byte_num; @@ -1398,34 +1314,7 @@ static void proxy_http_ctx_free(struct proxy_http_ctx * ctx) maat_stream_free(ctx->sp); ctx->sp=NULL; } - if(ctx->cache_write_ctx) - { - web_cache_write_end(ctx->cache_write_ctx); - ctx->cache_write_ctx=NULL; - } - //On session recycle - if(ctx->cache_revalidate_req) - { - tfe_http_half_free(ctx->cache_revalidate_req); - ctx->cache_revalidate_req=NULL; - } - if(ctx->cached_response) - { - //Dirty close - ctx->cached_response=NULL; - } - - if(ctx->f_cache_query) - { - future_destroy(ctx->f_cache_query); - ctx->f_cache_query=NULL; - } - if(ctx->f_cache_pending) - { - future_destroy(ctx->f_cache_pending); - ctx->f_cache_pending=NULL; - } if(ctx->log_req_body) { evbuffer_free(ctx->log_req_body); @@ -1472,7 +1361,7 @@ static unsigned long long try_send_by_token(int inject_sz) { return 1; } - return Ratelimiter_customer_factory(g_proxy_rt->ratelimiter, inject_sz); + return inject_sz; } static int pangu_action_weight[__PX_ACTION_MAX] = {0}; @@ -2875,248 +2764,6 @@ void enforce_control_policy(const struct tfe_stream * stream, const struct tfe_h return; } #define RESUMED_CB_NO_MORE_CALLS 0 -#define RESUMED_CB_MORE_CALLS 1 -int make_revalidate_request(const struct tfe_stream * stream, const struct tfe_http_session * session, - enum tfe_http_event events, const unsigned char * body_frag, size_t frag_size, unsigned int thread_id, struct proxy_http_ctx * ctx) -{ - assert(ctx->cache_revalidate_req); - if(events & EV_HTTP_REQ_BODY_BEGIN) - { - tfe_http_half_write_body_begin(ctx->cache_revalidate_req, 1); - } - if(events & EV_HTTP_REQ_BODY_CONT) - { - tfe_http_half_write_body_data(ctx->cache_revalidate_req, body_frag, frag_size); - } - if(events & EV_HTTP_REQ_BODY_END) - { - tfe_http_half_write_body_end(ctx->cache_revalidate_req); - ctx->cache_revalidate_req=NULL; - return RESUMED_CB_NO_MORE_CALLS; - } - if(events & EV_HTTP_REQ_END && ctx->cache_revalidate_req) - { - ctx->cache_revalidate_req=NULL; - return RESUMED_CB_NO_MORE_CALLS; - } - return RESUMED_CB_MORE_CALLS; -} -int dummy_resume(const struct tfe_stream * stream, const struct tfe_http_session * session, - enum tfe_http_event events, const unsigned char * body_frag, size_t frag_size, unsigned int thread_id, struct proxy_http_ctx * ctx) -{ - return RESUMED_CB_NO_MORE_CALLS; -} - -static void cache_read_on_succ(future_result_t * result, void * user) -{ - struct proxy_http_ctx * ctx = (struct proxy_http_ctx *)user; - const struct cached_meta* meta=NULL; - enum cache_query_result_type type=cache_query_result_get_type(result); - const unsigned char* data=NULL; - size_t data_sz=0; - char temp[TFE_STRING_MAX]; - - switch(type) - { - case CACHE_QUERY_RESULT_META: - meta=cache_query_result_read_meta(result); - ctx->cache_result_declared_sz=meta->content_length; - ctx->resumed_cb=dummy_resume; - tfe_http_session_resume(ctx->ref_session); - ATOMIC_DEC(&(g_proxy_rt->stat_val[STAT_SUSPENDING])); - - ctx->cached_response=tfe_http_session_response_create(ctx->ref_session, 200); - tfe_http_std_field_write(ctx->cached_response, TFE_HTTP_CONT_TYPE, meta->content_type); - tfe_http_std_field_write(ctx->cached_response, TFE_HTTP_LAST_MODIFIED, meta->last_modified); - tfe_http_std_field_write(ctx->cached_response, TFE_HTTP_ETAG, meta->etag); - - tfe_http_nonstd_field_write(ctx->cached_response, "X-TG-Cache-Lookup", "HIT"); - snprintf(temp, sizeof(temp), "%lu", meta->content_length); - tfe_http_std_field_write(ctx->cached_response, TFE_HTTP_CONT_LENGTH, temp); - - //Dirty code here. - tfe_http_session_response_set(ctx->ref_session, ctx->cached_response); - //From now, ownership of cached_response has been transfered to http session, - //bussines plugin only hold this pointer as an reference for writing response body. - tfe_http_half_write_body_begin(ctx->cached_response, 1); - - meta=NULL; - break; - case CACHE_QUERY_RESULT_DATA: - data_sz=cache_query_result_get_data(result, &data); - tfe_http_half_write_body_data(ctx->cached_response, data, data_sz); - ctx->cache_result_actual_sz+=data_sz; - break; - case CACHE_QUERY_RESULT_END: - assert(ctx->cached_response!=NULL); - tfe_http_half_write_body_end(ctx->cached_response); - //ownership has been transferred to http session, set to NULL. - ctx->pending_result=PENDING_RESULT_HIT; - ctx->cached_response=NULL; - assert(ctx->cache_result_actual_sz==ctx->cache_result_declared_sz); - future_destroy(ctx->f_cache_query); - ctx->f_cache_query=NULL; - break; - case CACHE_QUERY_RESULT_MISS: - ctx->pending_result=PENDING_RESULT_MISS; - ctx->resumed_cb=dummy_resume; - tfe_http_session_resume(ctx->ref_session); - ATOMIC_DEC(&(g_proxy_rt->stat_val[STAT_SUSPENDING])); - future_destroy(ctx->f_cache_query); - ctx->f_cache_query=NULL; - break; - default: - break; - } - return; -} -static void cache_read_on_fail(enum e_future_error err, const char * what, void * user) -{ - struct proxy_http_ctx * ctx = (struct proxy_http_ctx *)user; - future_destroy(ctx->f_cache_query); - ctx->f_cache_query=NULL; - if(!ctx->cached_response) - { - tfe_http_session_resume(ctx->ref_session); - ctx->resumed_cb=dummy_resume; - ATOMIC_DEC(&(g_proxy_rt->stat_val[STAT_SUSPENDING])); - } - else - { - tfe_http_half_write_body_end(ctx->cached_response); - ctx->cached_response=NULL; - } - ctx->pending_result=PENDING_RESULT_MISS; - printf("cache query failed: %s %s\n", ctx->ref_session->req->req_spec.url, what); -} -static void cache_pend_on_succ(future_result_t * result, void * user) -{ - struct proxy_http_ctx * ctx = (struct proxy_http_ctx *)user; - const struct cached_meta* meta=NULL; - meta=cache_pending_result_read_meta(result, ctx->cmid); - ctx->resumed_cb=dummy_resume; - tfe_http_session_resume(ctx->ref_session); - ATOMIC_DEC(&(g_proxy_rt->stat_val[STAT_SUSPENDING])); - future_destroy(ctx->f_cache_pending); - ctx->f_cache_pending=NULL; - if(meta==NULL) - { - ctx->pending_result = PENDING_RESULT_MISS; - return; - } - if( meta->etag==NULL && meta->last_modified==NULL) - { - ctx->pending_result = PENDING_RESULT_MISS; - return; - } - ctx->pending_result=PENDING_RESULT_REVALIDATE; - struct http_field_name in_field_name; - const char * in_header_value = NULL; - void * iterator = NULL; - ctx->cache_revalidate_req=tfe_http_session_request_create(ctx->ref_session, - ctx->ref_session->req->req_spec.method, ctx->ref_session->req->req_spec.uri); - while (true) - { - if ((in_header_value = tfe_http_field_iterate(ctx->ref_session->req, &iterator, &in_field_name)) == NULL) - { - break; - } - if(in_field_name.field_id==TFE_HTTP_IF_MATCH || in_field_name.field_id==TFE_HTTP_IF_NONE_MATCH - || in_field_name.field_id==TFE_HTTP_IF_MODIFIED_SINCE - || in_field_name.field_id==TFE_HTTP_IF_UNMODIFIED_SINCE) - { - continue; - } - tfe_http_field_write(ctx->cache_revalidate_req, &in_field_name, in_header_value); - } - if(meta->etag) tfe_http_std_field_write(ctx->cache_revalidate_req, TFE_HTTP_IF_NONE_MATCH, meta->etag); - if(meta->last_modified) tfe_http_std_field_write(ctx->cache_revalidate_req, TFE_HTTP_IF_MODIFIED_SINCE, meta->last_modified); - - tfe_http_session_request_set(ctx->ref_session, ctx->cache_revalidate_req); - ctx->resumed_cb=make_revalidate_request; - return; -} -static void cache_pend_on_fail(enum e_future_error err, const char * what, void * user) -{ - struct proxy_http_ctx * ctx = (struct proxy_http_ctx *)user; - - ctx->pending_result=PENDING_RESULT_FOBIDDEN; - tfe_http_session_resume(ctx->ref_session); - ATOMIC_DEC(&(g_proxy_rt->stat_val[STAT_SUSPENDING])); - ctx->resumed_cb=dummy_resume; - future_destroy(ctx->f_cache_pending); - ctx->f_cache_pending=NULL; - - return; -} - -void cache_pend(const struct tfe_http_session * session, unsigned int thread_id, struct proxy_http_ctx * ctx) -{ - if(g_proxy_rt->stat_val[STAT_SUSPENDING]>g_proxy_rt->suspend_max) - { - ctx->pending_result=PENDING_RESULT_FOBIDDEN; - return; - } - ctx->f_cache_pending=future_create("cache_pend", cache_pend_on_succ, cache_pend_on_fail, ctx); - ctx->ref_session=tfe_http_session_allow_write(session); - ctx->pending_result=web_cache_async_pending(g_proxy_rt->cache, thread_id, session->req, &(ctx->cmid), ctx->f_cache_pending); - switch(ctx->pending_result) - { - case PENDING_RESULT_REVALIDATE: - tfe_http_session_suspend(ctx->ref_session); - ATOMIC_INC(&(g_proxy_rt->stat_val[STAT_SUSPENDING])); - break; - case PENDING_RESULT_ALLOWED: - case PENDING_RESULT_FOBIDDEN: - case PENDING_RESULT_MISS: - future_destroy(ctx->f_cache_pending); - ctx->f_cache_pending=NULL; - break; - default: - break; - } - return; -} -void cache_read(const struct tfe_http_session * session, unsigned int thread_id, struct proxy_http_ctx * ctx) -{ - if(g_proxy_rt->stat_val[STAT_SUSPENDING]>g_proxy_rt->suspend_max) - { - return; - } - ctx->f_cache_query=future_create("cache_read", cache_read_on_succ, cache_read_on_fail, ctx); - int ret=web_cache_async_read(g_proxy_rt->cache, thread_id, session->req, &(ctx->cmid), ctx->f_cache_query); - if(ret==0) - { - ctx->ref_session=tfe_http_session_allow_write(session); - tfe_http_session_suspend(ctx->ref_session); - ATOMIC_INC(&(g_proxy_rt->stat_val[STAT_SUSPENDING])); - } - else - { - future_destroy(ctx->f_cache_query); - ctx->f_cache_query=NULL; - } -} - -void cache_write(const struct tfe_http_session * session, enum tfe_http_event events, - const unsigned char * body_frag, size_t frag_size, - unsigned int thread_id, struct proxy_http_ctx * ctx) -{ - - if(events & EV_HTTP_RESP_BODY_BEGIN) - { - ctx->cache_write_ctx=web_cache_write_start(g_proxy_rt->cache, thread_id, session, &(ctx->cmid)); - } - if(events & EV_HTTP_RESP_BODY_CONT && ctx->cache_write_ctx!=NULL) - { - web_cache_write(ctx->cache_write_ctx, body_frag, frag_size); - } - if(events & EV_HTTP_RESP_BODY_END && ctx->cache_write_ctx!=NULL) - { - ctx->cache_wirte_result=web_cache_write_end(ctx->cache_write_ctx); - ctx->cache_write_ctx=NULL; - } -} void proxy_on_http_begin(const struct tfe_stream *stream, const struct tfe_http_session *session, unsigned int thread_id, void **pme) { @@ -3387,11 +3034,6 @@ void proxy_on_http_end(const struct tfe_stream * stream, ATOMIC_INC(&(g_proxy_rt->stat_val[STAT_ACTION_RUN_SCRIPT])); } - TFE_LOG_DEBUG(g_proxy_rt->local_logger, "cache %s %s upload=%d", - session->req->req_spec.url, - cache_pending_result_string(ctx->pending_result), - ctx->cache_wirte_result); - cache_mid_clear(&(ctx->cmid)); proxy_http_ctx_free(ctx); *pme = NULL; return; @@ -3418,30 +3060,6 @@ int proxy_on_http_data(const struct tfe_stream * stream, const struct tfe_http_s } enforce_control_policy(stream, session, events, body_frag, frag_size,thread_id, ctx); - - if(g_proxy_rt->cache_enabled && ctx->action == PX_ACTION_NONE) - { - if(events & EV_HTTP_REQ_HDR) - { - cache_pend(session, thread_id, ctx); - if(ctx->pending_result==PENDING_RESULT_ALLOWED) - { - cache_read(session, thread_id, ctx); - } - } - if(events & EV_HTTP_RESP_HDR && ctx->pending_result==PENDING_RESULT_REVALIDATE) - { - if(session->resp->resp_spec.resp_code==TFE_HTTP_STATUS_NOT_MODIFIED) - { - cache_read(session, thread_id, ctx); - } - } - - if(tfe_http_in_response(events) && ctx->pending_result==PENDING_RESULT_MISS) - { - cache_write(session, events, body_frag, frag_size, thread_id, ctx); - } - } return NO_CALL_NEXT_PLUGIN; } diff --git a/plugin/business/tsg-http/src/tsg_logger.cpp b/plugin/business/tsg-http/src/tsg_logger.cpp index 9df6da1..2321306 100644 --- a/plugin/business/tsg-http/src/tsg_logger.cpp +++ b/plugin/business/tsg-http/src/tsg_logger.cpp @@ -1,10 +1,12 @@ #include <cjson/cJSON.h> #include <MESA/MESA_prof_load.h> #include <tfe_kafka_logger.h> -#include <cache_evbase_client.h> #include <tfe_utils.h> #include <tfe_resource.h> +#include <event2/event.h> +#include <event.h> + #include "mpack.h" #include "tsg_proxy_logger.h" @@ -16,7 +18,6 @@ struct json_spec struct proxy_logger { int entry_id; - unsigned int en_hoslog; unsigned int en_sendlog; const char *device_id; const char *effective_device_tag; @@ -113,8 +114,6 @@ struct proxy_logger* proxy_log_handle_create(const char* profile, const char* se instance->local_logger=local_logger; TFE_LOG_INFO(local_logger,"Tsg-Pxy log is inititating from %s section %s.", profile, section); - MESA_load_profile_int_def(profile, section, "ENTRANCE_ID",&(instance->entry_id),0); - MESA_load_profile_uint_def(profile, section, "en_hoslog", &instance->en_hoslog, 1); MESA_load_profile_uint_def(profile, section, "en_sendlog", &instance->en_sendlog, 1); TFE_LOG_INFO(local_logger, "Tsg-Pxy sendlog : %s", instance->en_sendlog ? "ENABLE" : "DISABLE"); @@ -132,16 +131,6 @@ struct proxy_logger* proxy_log_handle_create(const char* profile, const char* se goto error_out; } - if(instance->en_hoslog==1) - { - log_file_upload_para=cache_evbase_parameter_new(profile, section, local_logger); - if (log_file_upload_para == NULL) - { - TFE_LOG_ERROR(local_logger, "Tsg-Pxy failed to new cache evbase parameter."); - goto error_out; - } - instance->log_file_upload_instance=cache_evbase_instance_new(log_file_upload_para, local_logger); - } return instance; error_out: @@ -331,7 +320,6 @@ int proxy_send_log(struct proxy_logger* handle, const struct proxy_log* log_msg) for(size_t i=0; i<log_msg->result_num; i++) { if(log_msg->result[i].do_log!=1) continue; - if(handle->en_hoslog!=1) continue; if(log_msg->req_body!=NULL) { diff --git a/plugin/business/tsg-http/src/tsg_web_cache.cpp b/plugin/business/tsg-http/src/tsg_web_cache.cpp deleted file mode 100644 index 5f2d10e..0000000 --- a/plugin/business/tsg-http/src/tsg_web_cache.cpp +++ /dev/null @@ -1,1450 +0,0 @@ - -#include "tsg_proxy_web_cache.h" -#include <tango_cache_pending.h> -#include <tango_cache_client.h> - -#include <tfe_proxy.h> -#include <tfe_http.h> -#include <tfe_utils.h> - -#include <MESA/MESA_prof_load.h> -#include <MESA/field_stat2.h> - -#include <event2/event.h> -#include <event2/buffer.h> - -extern "C" -{ -#include <dablooms.h> -} - -#include <cjson/cJSON.h> - -#include <pthread.h> - -enum cache_stat_field -{ - STAT_CACHE_READ, - STAT_CACHE_PEND_FORBIDDEN, - STAT_CACHE_READ_VERIFY, - STAT_CACHE_READ_HIT, - STAT_CACHE_READ_BYTES, - STAT_CACHE_OVERRIDE_READ, - STAT_CACHE_OVERRIDE_READ_HIT, - STAT_CACHE_OVERRIDE_READ_BYTES, - STAT_CACHE_READ_ERR, - STAT_CACHE_READ_THROTTLE, - STAT_CACHE_READING, - STAT_CACHE_PENDING, - STAT_CACHE_POLICY_MATCH, - STAT_CACHE_POLICY_EFFECT, - STAT_CACHE_WRITE_CNT, - STAT_CACHE_WRITE_BYPASS, - STAT_CACHE_OVERRIDE_WRITE, - STAT_CACHE_WRITE_FORBIDEN, - STAT_CACHE_WRITE_THROTTLE, - STAT_CACHE_WRITE_CANCEL, - STAT_CACHE_WRITE_ERR, - STAT_CACHE_WRITE_BYTES, - STAT_CACHE_WRITING, - STAT_CACHE_MEMORY, - STAT_CACHE_ACTIVE_SESSION_HTTP, - STAT_CACHE_ACTIVE_SESSION_REDIS, - STAT_CACHE_QUERY_HIT_OJB_SIZE, - STAT_CACHE_WRITE_OBJ_SIZE, - STAT_CACHE_OVERRIDE_HIT_OBJ_SIZE, - STAT_CACHE_OVERRIDE_WRITE_OBJ_SIZE, - __CACHE_STAT_MAX -}; - -struct cache_key_descr -{ - int is_not_empty; - size_t qs_num; - char** ignore_qs; - char* include_cookie; -}; -struct cache_param -{ - int ref_cnt; - struct cache_key_descr key_descr; - - char no_revalidate; - char cache_dyn_url; - char cache_html; - char cache_cookied_cont; - char ignore_req_nocache; - char ignore_res_nocache; - char force_caching; - - long long config_id; - int action; - int min_use; - time_t pinning_time_sec; - time_t inactive_time_sec; - size_t max_cache_size; - size_t max_cache_obj_size; - size_t min_cache_obj_size; - pthread_mutex_t lock; -}; -struct cache_bloom -{ - int thread_id; - size_t size; - double error_rate; - char filename[TFE_PATH_MAX]; - counting_bloom_t *bloom; -}; -struct cache_handle -{ - unsigned int thread_count; - int cache_undefined_obj_enabled; - int query_undefined_obj_enabled; - size_t cache_undefined_obj_min_size; - int minimum_cache_seconds; - struct tango_cache_instance **clients; - - long long get_concurrency_max; - long long put_concurrency_max; - - screen_stat_handle_t fs_handle; - long long stat_val[__CACHE_STAT_MAX]; - int fs_id[__CACHE_STAT_MAX]; - struct event_base* gc_evbase; - struct event* gcev; - - - int cache_policy_enabled; //otherwise use default cache policy - struct cache_param default_cache_policy; - struct maat *ref_feather; - long long cache_param_idx; - int table_url_constraint; - int table_cookie_constraint; - - int cache_key_bloom_life; - size_t cache_key_bloom_size; - struct cache_bloom *cache_key_bloom; - void* logger; -}; -struct cache_write_context -{ - struct cache_write_future_ctx* future_ctx; - struct cache_handle* ref_cache_handle; - struct tango_cache_ctx * write_ctx; - size_t content_len; - size_t uploaded_len; -}; -const char* cache_pending_result_string(enum cache_pending_result result) -{ - switch (result) - { - case PENDING_RESULT_NONE: - return "none"; - case PENDING_RESULT_FOBIDDEN: - return "forbidden"; - case PENDING_RESULT_REVALIDATE: - return "revalidate"; - case PENDING_RESULT_ALLOWED: - return "allowd"; - case PENDING_RESULT_HIT: - return "hit"; - case PENDING_RESULT_MISS: - return "miss"; - default: - assert(0); - return "?"; - } - -} - -static void web_cache_stat_cb(evutil_socket_t fd, short what, void * arg) -{ - struct cache_handle* cache=(struct cache_handle *)arg; - struct cache_statistics client_stat_sum, client_stat; - memset(&client_stat_sum, 0, sizeof(client_stat_sum)); - long long *val_sum = (long long *)&client_stat_sum; - long long *val = NULL; - unsigned int i=0, j=0; - for(i=0; i<cache->thread_count;i++) - { - tango_cache_get_statistics(cache->clients[i], &client_stat); - val=(long long*)&client_stat; - for(j=0; j<sizeof(client_stat)/sizeof(long long); j++) - { - val_sum[j]+=val[j]; - } - } - for(i=0;i<__CACHE_STAT_MAX;i++) - { - if(ATOMIC_READ(&(cache->stat_val[i]))!=0) - { - switch(i) - { - case STAT_CACHE_WRITE_BYTES: - case STAT_CACHE_READ_BYTES: - case STAT_CACHE_OVERRIDE_READ_BYTES: - //translate bytes to mega bytes. - FS_operate(cache->fs_handle, cache->fs_id[i], 0, FS_OP_SET, ATOMIC_READ(&(cache->stat_val[i]))/(1024*1024)); - break; - default: - FS_operate(cache->fs_handle, cache->fs_id[i], 0, FS_OP_SET, ATOMIC_READ(&(cache->stat_val[i]))); - break; - } - } - } - - FS_operate(cache->fs_handle, cache->fs_id[STAT_CACHE_READ], 0, FS_OP_SET, client_stat_sum.get_recv_num); - FS_operate(cache->fs_handle, cache->fs_id[STAT_CACHE_READ_HIT], 0, - FS_OP_SET, client_stat_sum.get_succ_http+client_stat_sum.get_succ_redis); - FS_operate(cache->fs_handle, cache->fs_id[STAT_CACHE_READ_ERR], 0, - FS_OP_SET, client_stat_sum.get_err_http+client_stat_sum.get_err_redis); - FS_operate(cache->fs_handle, cache->fs_id[STAT_CACHE_WRITE_CNT], 0, - FS_OP_SET, client_stat_sum.put_recv_num); - FS_operate(cache->fs_handle, cache->fs_id[STAT_CACHE_WRITE_ERR], 0, - FS_OP_SET, client_stat_sum.put_err_http+client_stat_sum.put_err_redis); - FS_operate(cache->fs_handle, cache->fs_id[STAT_CACHE_MEMORY], 0, FS_OP_SET, client_stat_sum.memory_used/(1024*1024)); - FS_operate(cache->fs_handle, cache->fs_id[STAT_CACHE_ACTIVE_SESSION_HTTP], 0, FS_OP_SET, client_stat_sum.session_http); - FS_operate(cache->fs_handle, cache->fs_id[STAT_CACHE_WRITE_THROTTLE], 0, FS_OP_SET, client_stat_sum.totaldrop_num); - FS_passive_output(cache->fs_handle); - return; -} -struct cache_stat_sepc -{ - const char* name; - enum field_dsp_style_t style; - enum field_calc_algo calc_type; -}; - -static void set_stat_spec(struct cache_stat_sepc* spec, const char* name, enum field_dsp_style_t style, enum field_calc_algo calc_type) -{ - spec->name=name; - spec->style=style; - spec->calc_type=calc_type; - return; -} -void cache_stat_init(struct cache_handle* cache, -const char* statsd_server_ip, int statsd_server_port, const char*histogram_bins) -{ - const char* fieldstat_output="log/cache.fs2"; - const char* app_name="tfe_cache"; - - int value=0, i=0; - screen_stat_handle_t fs_handle=NULL; - fs_handle=FS_create_handle(); - FS_set_para(fs_handle, OUTPUT_DEVICE, fieldstat_output, strlen(fieldstat_output)+1); - value=1; - FS_set_para(fs_handle, PRINT_MODE, &value, sizeof(value)); - value=0; - FS_set_para(fs_handle, CREATE_THREAD, &value, sizeof(value)); - FS_set_para(fs_handle, APP_NAME, app_name, strlen(app_name)+1); - FS_set_para(fs_handle, HISTOGRAM_GLOBAL_BINS, histogram_bins, strlen(histogram_bins)+1); - if(strlen(statsd_server_ip)>0 && statsd_server_port!=0) - { - FS_set_para(fs_handle, STATS_SERVER_IP, statsd_server_ip, strlen(statsd_server_ip)+1); - FS_set_para(fs_handle, STATS_SERVER_PORT, &(statsd_server_port), sizeof(statsd_server_port)); - } - - cache->fs_handle=fs_handle; - - struct cache_stat_sepc spec[__CACHE_STAT_MAX]; - - set_stat_spec(&spec[STAT_CACHE_READ], "cache_read",FS_STYLE_FIELD, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_PEND_FORBIDDEN], "read_forbid",FS_STYLE_FIELD, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_READ_VERIFY], "read_verify",FS_STYLE_FIELD, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_READ_HIT], "hit_num",FS_STYLE_FIELD, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_READ_BYTES], "read(MB)",FS_STYLE_FIELD, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_OVERRIDE_READ], "or_read",FS_STYLE_FIELD, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_OVERRIDE_READ_HIT], "or_hit",FS_STYLE_FIELD, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_OVERRIDE_READ_BYTES], "or_read(MB)",FS_STYLE_FIELD, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_READ_ERR], "read_err",FS_STYLE_STATUS, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_READ_THROTTLE], "read_throt",FS_STYLE_STATUS, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_READING], "reading",FS_STYLE_STATUS, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_PENDING], "pending",FS_STYLE_STATUS, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_POLICY_MATCH], "ply_match",FS_STYLE_FIELD, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_POLICY_EFFECT], "ply_effect",FS_STYLE_FIELD, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_WRITE_CNT], "cache_write",FS_STYLE_FIELD, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_WRITE_BYPASS], "write_bypass",FS_STYLE_FIELD, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_OVERRIDE_WRITE], "or_write",FS_STYLE_FIELD, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_WRITE_FORBIDEN], "write_forbid",FS_STYLE_FIELD, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_WRITE_THROTTLE], "write_throt",FS_STYLE_FIELD, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_WRITE_CANCEL], "write_cancel",FS_STYLE_FIELD, FS_CALC_CURRENT); - - set_stat_spec(&spec[STAT_CACHE_WRITE_ERR], "write_err",FS_STYLE_STATUS, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_WRITE_BYTES], "write(MB)",FS_STYLE_FIELD, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_WRITING], "writing",FS_STYLE_STATUS, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_MEMORY], "buffer(MB)",FS_STYLE_STATUS, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_ACTIVE_SESSION_HTTP], "sess_http",FS_STYLE_STATUS, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_ACTIVE_SESSION_REDIS], "sess_redis",FS_STYLE_STATUS, FS_CALC_CURRENT); - - set_stat_spec(&spec[STAT_CACHE_OVERRIDE_HIT_OBJ_SIZE], "or_hit_obj(KB)",FS_STYLE_HISTOGRAM, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_QUERY_HIT_OJB_SIZE], "hit_obj_sz(KB)",FS_STYLE_HISTOGRAM, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_WRITE_OBJ_SIZE], "wr_obj_sz(KB)",FS_STYLE_HISTOGRAM, FS_CALC_CURRENT); - set_stat_spec(&spec[STAT_CACHE_OVERRIDE_WRITE_OBJ_SIZE], "or_obj_sz(KB)",FS_STYLE_HISTOGRAM, FS_CALC_CURRENT); - - - for(i=0;i<__CACHE_STAT_MAX;i++) - { - if(spec[i].style==FS_STYLE_HISTOGRAM) - { - cache->fs_id[i]=FS_register_histogram(cache->fs_handle, spec[i].calc_type, spec[i].name,1,10*1024*1024,2); - } - else - { - cache->fs_id[i]=FS_register(cache->fs_handle, spec[i].style, spec[i].calc_type, spec[i].name); - } - } -// value=cache->fs_id[STAT_CACHE_READ_HIT]; -// FS_set_para(cache->fs_handle, ID_INVISBLE, &value, sizeof(value)); - - FS_register_ratio(cache->fs_handle, - cache->fs_id[STAT_CACHE_READ_HIT], - cache->fs_id[STAT_CACHE_READ], - 1, - FS_STYLE_STATUS, - FS_CALC_CURRENT, - "cache_hit"); - - value=cache->fs_id[STAT_CACHE_OVERRIDE_READ_HIT]; - FS_set_para(cache->fs_handle, ID_INVISBLE, &value, sizeof(value)); - - FS_register_ratio(cache->fs_handle, - cache->fs_id[STAT_CACHE_OVERRIDE_READ_HIT], - cache->fs_id[STAT_CACHE_OVERRIDE_READ], - 1, - FS_STYLE_STATUS, - FS_CALC_CURRENT, - "or_hit"); - - - FS_start(cache->fs_handle); - - struct timeval gc_delay = {0, 500*1000}; //Microseconds, we set 500 miliseconds here. - cache->gcev = event_new(cache->gc_evbase, -1, EV_PERSIST, web_cache_stat_cb, cache); - evtimer_add(cache->gcev, &gc_delay); - return; -} - - -time_t time_unit_sec(const char* str) -{ - time_t value=0; - sscanf(str, "%ld", &value); - switch(str[strlen(str)-1]) - { - case 's': - break; - case 'm': - value*=60; - break; - case 'h': - value*=3600; - break; - case 'd': - value*=((size_t)24*3600); - break; - default: - break; - } - return value; -} -size_t storage_unit_byte(const char* str) -{ - size_t value=0; - sscanf(str, "%ld", &value); - switch(str[strlen(str)-1]) - { - case 'b': - break; - case 'k': - value*=1024; - break; - case 'm': - value*=((size_t)1024*1024); - break; - case 'g': - value*=((size_t)1024*1024*1024); - break; - case 't': - if(value<1024) - { -#pragma GCC diagnostic ignored "-Woverflow" - value*=((size_t)1024*1024*1024*1024); - - } - else //maximum 1PB - { - value=(size_t)1024*(1024*1024*1024*1024); - } - break; - default: - break; - } - return value; -} - - -//A URL is considered dynamic if it ends in “.asp(x)” or contains a question mark (?), a semicolon (;), or “cgi”. -char is_dynamic_url(const char* url) -{ - if(strchr(url, '?') - || strchr(url, ';') - || strstr(url, "cgi") - || 0==strcmp(url+strlen(url)-3,"asp") - || 0==strcmp(url+strlen(url)-4, "aspx")) - { - return 1; - } - return 0; -} - -char * cookie_scanvalue(const char * key, const char * cookies, char * val, size_t val_len) -{ - unsigned int i=0, j=0, k=0; - int found=1; - char* key_dup=ALLOC(char, strlen(key)+2); - char* cookie_dup=ALLOC(char, strlen(cookies)+1); - strcat(key_dup, key); - if(key_dup[strlen(key)]!='=') - { - strcat(key_dup, "="); - } - for(i=0; i<strlen(cookies); i++) - { - if(cookies[i]==' '||cookies[i]=='\t'||cookies[i]=='\r'||cookies[i]=='\n') - { - continue; - } - cookie_dup[j]=cookies[i]; - j++; - } - char *token=NULL,*sub_token=NULL,*saveptr; - - for (token = cookie_dup; ; token= NULL) - { - sub_token= strtok_r(token,";", &saveptr); - if (sub_token == NULL) - break; - if(0==strncasecmp(sub_token, key_dup, strlen(key_dup))) - { - found=1; - sub_token+=strlen(key_dup); - k = strcspn(sub_token, ";"); - strncpy(val, sub_token, MIN(val_len-1, k+1)); - - break; - } - } - if(!found &&val_len > 0) - { - val[0] = '\0'; - } - FREE(&key_dup); - FREE(&cookie_dup); - return val; -} -char* url_remove_qs(const char* url, int qs_num, char* ignore_qs[]) -{ - char* url_copy=tfe_strdup(url); - size_t target_size= strlen(url_copy)+1; - char* target_url=ALLOC(char, target_size); - int i=0, shall_ignore=0; - char *token=NULL,*sub_token=NULL,*saveptr; - - char* query_string=NULL; - query_string=strchr(url_copy, '?'); - if(query_string!=NULL) - { - strncat(target_url, url_copy, MIN((unsigned int)(query_string-url_copy),target_size)); - query_string++; - for (token = query_string; ; token= NULL) - { - sub_token= strtok_r(token,"&", &saveptr); - if (sub_token == NULL) - break; - shall_ignore=0; - for(i=0; i<qs_num; i++) - { - if(0==strncasecmp(sub_token, ignore_qs[i], strlen(ignore_qs[i]))) - { - shall_ignore=1; - break; - } - } - if(!shall_ignore) - { - strncat(target_url, sub_token, target_size); - } - } - } - else - { - strncat(target_url, url_copy, target_size); - } - - FREE(&(url_copy)); - return target_url; -} - -char* get_cache_key(const struct tfe_http_half * request, const struct cache_key_descr* desc) -{ - if(desc==NULL|| !desc->is_not_empty) - { - return NULL; - } - char* url_no_qs=NULL; - const char* cookie=NULL; - char cookie_val[1024]={0}; //most 1024 bytes for cookie key - - size_t key_size=strlen(request->req_spec.url)+sizeof(cookie_val); - char* cache_key=ALLOC(char, key_size); - - if(desc->qs_num>0) - { - url_no_qs=url_remove_qs(request->req_spec.url, desc->qs_num, desc->ignore_qs); - strncat(cache_key, url_no_qs, key_size); - FREE(&url_no_qs); - } - else - { - strncat(cache_key, request->req_spec.url, key_size); - } - if(desc->include_cookie && (cookie=tfe_http_std_field_read(request, TFE_HTTP_COOKIE))!=NULL) - { - cookie_scanvalue(desc->include_cookie, cookie, cookie_val, sizeof(cookie_val)); - if(strlen(cookie_val)>0) - { - snprintf(cache_key, key_size-strlen(cache_key), "/C/%s=%s", desc->include_cookie, cookie_val); - } - } - return cache_key; -} - -void cache_param_new(const char *table_name, int idx, const char *key, const char *table_line, void **ad, long argl, void *argp) -{ - struct cache_handle* cache=(struct cache_handle*) argp; - unsigned int i=0; - size_t len=0; - int ret=0; - int config_id=0, service_id=0, action=0; - int do_log=0,do_blacklist=0,is_valid=0; - char effective_range[1024]={0}; - char srv_def_large[8192]={0}; - - ret=sscanf(table_line, "%d\t%d\t%d\t%d\t%d\t%s\t%s\t%d", &config_id, &service_id, &action, &do_blacklist, &do_log, effective_range, srv_def_large, &is_valid); - if(ret!=8) - { - return; - } - - *ad=NULL; - int serv_def_len=strlen(srv_def_large); - if((unsigned int)serv_def_len<strlen("{}")+1) - { - return; - } - cJSON *json=NULL, *key_desc=NULL, *qs=NULL,*item=NULL; - json=cJSON_Parse(srv_def_large); - if(json==NULL) - { - TFE_LOG_ERROR(cache->logger, "invalid cache parameter: id = %d", config_id); - return; - } - struct cache_param* param=ALLOC(struct cache_param, 1); - *param=cache->default_cache_policy; - param->ref_cnt=1; - param->config_id=config_id; - param->action=action; - pthread_mutex_init(&(param->lock), NULL); - key_desc=cJSON_GetObjectItem(json,"cache_key"); - if(key_desc && key_desc->type==cJSON_Object) - { - qs=cJSON_GetObjectItem(key_desc,"ignore_qs"); - if(qs && qs->type==cJSON_Array) - { - param->key_descr.qs_num=cJSON_GetArraySize(qs); - param->key_descr.ignore_qs=ALLOC(char*, param->key_descr.qs_num); - for(i=0; i<param->key_descr.qs_num; i++) - { - item=cJSON_GetArrayItem(qs, i); - len=strlen(item->valuestring)+2; - param->key_descr.ignore_qs[i]=ALLOC(char, len); - strncat(param->key_descr.ignore_qs[i], item->valuestring, len); - strncat(param->key_descr.ignore_qs[i], "=", len); - } - } - item=cJSON_GetObjectItem(key_desc,"cookie"); - if(item && item->type==cJSON_String) - { - param->key_descr.include_cookie=tfe_strdup(item->valuestring); - - } - if(param->key_descr.qs_num>0||param->key_descr.include_cookie!=NULL) - { - param->key_descr.is_not_empty=1; - } - } - - item=cJSON_GetObjectItem(json,"no_revalidate"); - if(item && item->type==cJSON_Number) param->no_revalidate=item->valueint; - - item=cJSON_GetObjectItem(json,"cache_dyn_url"); - if(item && item->type==cJSON_Number) param->cache_dyn_url=item->valueint; - - item=cJSON_GetObjectItem(json,"cache_cookied_cont"); - if(item && item->type==cJSON_Number) param->cache_cookied_cont=item->valueint; - - item=cJSON_GetObjectItem(json,"ignore_req_nocache"); - if(item && item->type==cJSON_Number) param->ignore_req_nocache=item->valueint; - - item=cJSON_GetObjectItem(json,"ignore_res_nocache"); - if(item && item->type==cJSON_Number) param->ignore_res_nocache=item->valueint; - - item=cJSON_GetObjectItem(json,"force_caching"); - if(item && item->type==cJSON_Number) param->force_caching=item->valueint; - - item=cJSON_GetObjectItem(json,"min_use"); - if(item && item->type==cJSON_Number) param->min_use=item->valueint; - - item=cJSON_GetObjectItem(json,"pinning_time"); - if(item && item->type==cJSON_String) param->pinning_time_sec=time_unit_sec(item->valuestring); - - item=cJSON_GetObjectItem(json,"inactive_time"); - if(item && item->type==cJSON_String) param->inactive_time_sec=time_unit_sec(item->valuestring); - - item=cJSON_GetObjectItem(json,"max_cache_size"); - if(item && item->type==cJSON_String) param->max_cache_size=storage_unit_byte(item->valuestring); - - item=cJSON_GetObjectItem(json,"max_cache_obj_size"); - if(item && item->type==cJSON_String) param->max_cache_obj_size=storage_unit_byte(item->valuestring); - - item=cJSON_GetObjectItem(json,"min_cache_obj_size"); - if(item && item->type==cJSON_String) param->min_cache_obj_size=storage_unit_byte(item->valuestring); - - cJSON_Delete(json); - *ad=param; - return; -} -void cache_param_free(int table_id, void **ad, long argl, void *argp) -{ - unsigned int i=0; - if(*ad==NULL) - { - return; - } - struct cache_param* param=(struct cache_param*)*ad; - pthread_mutex_lock(&(param->lock)); - param->ref_cnt--; - if(param->ref_cnt>0) - { - pthread_mutex_unlock(&(param->lock)); - return; - } - pthread_mutex_unlock(&(param->lock)); - pthread_mutex_destroy(&(param->lock)); - for(i=0; i<param->key_descr.qs_num; i++) - { - FREE(&(param->key_descr.ignore_qs[i])); - } - FREE(&(param->key_descr.ignore_qs)); - FREE(&(param->key_descr.include_cookie)); - FREE(&(param)); - return; -} -void cache_param_dup(int idx, void **to, void **from, long argl, void *argp) -{ - struct cache_param* from_param=*((struct cache_param**)from); - pthread_mutex_lock(&(from_param->lock)); - from_param->ref_cnt++; - pthread_mutex_unlock(&(from_param->lock)); - *((struct cache_param**)to)=from_param; - return; -} - -struct cache_mid -{ - enum cache_pending_result result; - enum OBJECT_LOCATION location; - struct request_freshness req_fresshness; - char shall_bypass; - char is_using_exception_param; - char is_dyn_url; - char is_html; - char has_cookie; - char use_cnt; - int cfg_id; - char* cache_key; - struct cache_param* param; -}; -void cache_mid_clear(struct cache_mid **mid) -{ - if(*mid==NULL) - { - return; - } - if((*mid)->is_using_exception_param) - { - cache_param_free(0, (void**)&((*mid)->param), 0, NULL); - } - FREE(&((*mid)->cache_key)); - FREE(mid); - return; -} - -static void cache_key_bloom_gc_cb(evutil_socket_t fd, short what, void * arg) -{ - struct cache_bloom* p_bloom= (struct cache_bloom*) arg; - counting_bloom_t* new_bloom=NULL; - - new_bloom=new_counting_bloom(p_bloom->size, p_bloom->error_rate, p_bloom->filename); - free_counting_bloom(p_bloom->bloom); - p_bloom->bloom=new_bloom; - return; -} - -struct cache_handle* create_web_cache_handle(const char* profile_path, const char* section, - struct event_base* gc_evbase, struct maat *feather, void *logger) -{ - struct cache_handle* cache=ALLOC(struct cache_handle, 1); - int temp=0; - struct event* ev=NULL; - char statsd_server_ip[TFE_SYMBOL_MAX]={0}; - char histogram_bins[TFE_SYMBOL_MAX]={0}; - int statsd_server_port=0; - - cache->logger=logger; - cache->thread_count=tfe_proxy_get_work_thread_count(); - cache->clients=ALLOC(struct tango_cache_instance *, cache->thread_count); - cache->cache_key_bloom=ALLOC(struct cache_bloom, cache->thread_count); - struct cache_bloom* p_bloom=NULL; - MESA_load_profile_int_def(profile_path, section, "cache_policy_enabled", - &(cache->cache_policy_enabled), 1); - - - MESA_load_profile_int_def(profile_path, section, "cache_key_bloom_size", - (int*)&(cache->cache_key_bloom_size), 16*1000*1000); - MESA_load_profile_int_def(profile_path, section, "cache_key_bloom_life", - &(cache->cache_key_bloom_life), 30*60); - struct timeval gc_refresh_delay = {cache->cache_key_bloom_life, 0}; - unsigned int i=0; - - struct tango_cache_parameter *cache_client_param=tango_cache_parameter_new(profile_path, section, logger); - for(i=0; i<cache->thread_count; i++) - { - if(cache->cache_policy_enabled) - { - p_bloom=cache->cache_key_bloom+i; - p_bloom->thread_id=i; - p_bloom->size=cache->cache_key_bloom_size; - p_bloom->error_rate=0.01; - snprintf(p_bloom->filename, sizeof(p_bloom->filename), "/tmp/pangu_cache_blooms.%02d", i); - p_bloom->bloom=new_counting_bloom(p_bloom->size, p_bloom->error_rate, p_bloom->filename); - if(p_bloom->bloom==NULL) - { - goto error_out; - } - ev = event_new(tfe_proxy_get_work_thread_evbase(i), -1, EV_PERSIST, cache_key_bloom_gc_cb, p_bloom); - evtimer_add(ev, &gc_refresh_delay); - } - - cache->clients[i]=tango_cache_instance_new(cache_client_param,tfe_proxy_get_work_thread_evbase(i), logger); - if(cache->clients[i]==NULL) - { - goto error_out; - } - } - - MESA_load_profile_int_def(profile_path, section, "get_concurrency_max", &temp, 1000*1000); - cache->get_concurrency_max=temp; - MESA_load_profile_int_def(profile_path, section, "put_concurrency_max", &(temp), 1000*1000); - cache->put_concurrency_max=temp; - MESA_load_profile_int_def(profile_path, section, "query_undefined_obj", &(cache->query_undefined_obj_enabled), 1); - MESA_load_profile_int_def(profile_path, section, "cache_undefined_obj", &(cache->cache_undefined_obj_enabled), 1); - MESA_load_profile_int_def(profile_path, section, "cached_undefined_obj_minimum_size", &(temp), 100*1024); - cache->cache_undefined_obj_min_size=temp; - - cache->gc_evbase=gc_evbase; - - cache->default_cache_policy.key_descr.qs_num=0; - cache->default_cache_policy.no_revalidate=0; - cache->default_cache_policy.cache_dyn_url=0; - cache->default_cache_policy.cache_cookied_cont=0; - cache->default_cache_policy.ignore_req_nocache=0; - cache->default_cache_policy.ignore_res_nocache=0; - cache->default_cache_policy.force_caching=0; - cache->default_cache_policy.pinning_time_sec=0; - cache->default_cache_policy.inactive_time_sec=0; - cache->default_cache_policy.max_cache_size=0; - - MESA_load_profile_int_def(profile_path, section, "min_use", &(cache->default_cache_policy.min_use), 0); - MESA_load_profile_int_def(profile_path, section, "max_cache_obj_size", &(temp), 1024*1024*1024); - cache->default_cache_policy.max_cache_obj_size=temp; //<1GB by default - - MESA_load_profile_int_def(profile_path, section, "min_cache_obj_size", &(temp), 1*1024); - cache->default_cache_policy.min_cache_obj_size=temp;// > 16kb by default - - if(cache->cache_policy_enabled) - { - cache->cache_param_idx=maat_get_table_id(feather, "PXY_CACHE_COMPILE_PLUGIN"); - cache->table_url_constraint=maat_get_table_id(feather, "PXY_CACHE_HTTP_URL"); - cache->table_cookie_constraint=maat_get_table_id(feather, "PXY_CACHE_HTTP_COOKIE"); - cache->cache_param_idx=maat_plugin_table_ex_schema_register(feather, "PXY_CACHE_COMPILE_PLUGIN", - cache_param_new, cache_param_free, cache_param_dup, - 0, cache); - cache->ref_feather=feather; - } - - MESA_load_profile_string_def(profile_path, section, "statsd_server", statsd_server_ip, - sizeof(statsd_server_ip), ""); - MESA_load_profile_int_def(profile_path, section, "statsd_port", &(statsd_server_port), 0); - MESA_load_profile_string_def(profile_path, section, "histogram_bins", - histogram_bins, sizeof(histogram_bins), "0.5,0.8,0.9,0.95"); - - cache_stat_init(cache, statsd_server_ip, statsd_server_port, histogram_bins); - return cache; -error_out: - free(cache); - return NULL; -} - -static char* read_http1_hdr(const char* hdr, const char* field_name) -{ - const char *p=NULL, *q=NULL; - char* value=NULL; - p=strcasestr(hdr, field_name); - if(p==NULL) - { - return NULL; - } - p=strstr(p, ":"); - if(p==NULL) - { - return NULL; - } - p++; - q=strcasestr(p, "\r\n"); - if(q==NULL) - { - return NULL; - } - value=(char*) calloc(sizeof(char), (q-p+1)); - memcpy(value, p, q-p); - return value; -} -struct cache_query_context -{ - struct cache_handle* ref_handle; - const struct cache_mid* ref_mid; - char* url; - struct cached_meta meta; - - struct tango_cache_result* ref_tango_cache_result; - struct future* f_tango_cache_fetch; -}; - -enum cache_query_result_type cache_query_result_get_type(future_result_t * result) -{ - struct cache_query_context* ctx=(struct cache_query_context*)result; - struct tango_cache_result* cache_result=ctx->ref_tango_cache_result; - enum cache_query_result_type map[__CACHE_QUERY_RESULT_MAX]; - map[RESULT_TYPE_BODY]=CACHE_QUERY_RESULT_DATA; - map[RESULT_TYPE_HEADER]=CACHE_QUERY_RESULT_META; - map[RESULT_TYPE_USERTAG]=CACHE_QUERY_RESULT_IRRELEVANT; - map[RESULT_TYPE_END]=CACHE_QUERY_RESULT_END; - map[RESULT_TYPE_MISS]=CACHE_QUERY_RESULT_MISS; - return map[cache_result->type]; -} - -size_t cache_query_result_get_data(future_result_t * result, const unsigned char** pp_data) -{ - struct cache_query_context* ctx=(struct cache_query_context*)result; - struct tango_cache_result* cache_result=ctx->ref_tango_cache_result; - assert(cache_result->type==RESULT_TYPE_BODY); - *pp_data=(const unsigned char*)cache_result->data_frag; - return cache_result->size; -} - -void cache_query_ctx_free_cb(void* p) -{ - struct cache_query_context* ctx=(struct cache_query_context*)p; - future_destroy(ctx->f_tango_cache_fetch); - ctx->f_tango_cache_fetch=NULL; - FREE(&(ctx->url)); - FREE(&(ctx->meta.etag)); - FREE(&(ctx->meta.last_modified)); - FREE(&(ctx->meta.content_type)); - free(ctx); -} -const struct cached_meta* cache_query_result_read_meta(future_result_t * result) -{ - struct cache_query_context* ctx=(struct cache_query_context*)result; - return &(ctx->meta); -} -void cached_meta_set(struct cached_meta* meta, enum CACHE_RESULT_TYPE type, const char* data_frag, size_t size) -{ - switch(type) - { - case RESULT_TYPE_HEADER: - meta->content_type=read_http1_hdr((const char*)data_frag, "content-type"); - break; - case RESULT_TYPE_USERTAG: - meta->last_modified=read_http1_hdr(data_frag, "Last-Modified"); - if(meta->last_modified!=NULL && 0==strcasecmp(meta->last_modified, "Thu, 01 Jan 1970 00:00:00 GMT")) - { - FREE(&(meta->last_modified)); - } - meta->etag=read_http1_hdr(data_frag, "etag"); - break; - default: - assert(0); - break; - } - return; -} - -static void cache_query_obj_on_succ(future_result_t * result, void * user) -{ - struct promise * p = (struct promise *) user; - struct cache_query_context* ctx=(struct cache_query_context*)promise_get_ctx(p); - int last_call=0; - ctx->ref_tango_cache_result=tango_cache_read_result(result); - switch(ctx->ref_tango_cache_result->type) - { - case RESULT_TYPE_HEADER: - ATOMIC_INC(&(ctx->ref_handle->stat_val[STAT_CACHE_READ_HIT])); - if(ctx->ref_mid->is_using_exception_param==1) - { - ATOMIC_INC(&(ctx->ref_handle->stat_val[STAT_CACHE_POLICY_EFFECT])); - } - FS_operate(ctx->ref_handle->fs_handle, ctx->ref_handle->fs_id[STAT_CACHE_QUERY_HIT_OJB_SIZE], 0, FS_OP_SET, ctx->ref_tango_cache_result->tlength/1024); - cached_meta_set(&ctx->meta, RESULT_TYPE_HEADER, ctx->ref_tango_cache_result->data_frag, ctx->ref_tango_cache_result->size); - ctx->meta.content_length=ctx->ref_tango_cache_result->tlength; - TFE_LOG_DEBUG(ctx->ref_handle->logger, "cache query hit: %s", ctx->url); - break; - case RESULT_TYPE_USERTAG: - cached_meta_set(&ctx->meta, RESULT_TYPE_USERTAG, ctx->ref_tango_cache_result->data_frag, ctx->ref_tango_cache_result->size); - break; - case RESULT_TYPE_MISS: - TFE_LOG_DEBUG(ctx->ref_handle->logger, "cache query miss: %s", ctx->url); - //NOT break intentionally. - case RESULT_TYPE_END: - //last call. - ATOMIC_DEC(&(ctx->ref_handle->stat_val[STAT_CACHE_READING])); - promise_dettach_ctx(p); - last_call=1; - break; - case RESULT_TYPE_BODY: - ATOMIC_ADD(&(ctx->ref_handle->stat_val[STAT_CACHE_READ_BYTES]), ctx->ref_tango_cache_result->size); - break; - default: - break; - } - promise_success(p, ctx); - if(last_call) - { - cache_query_ctx_free_cb(ctx); - promise_finish(p); - } - return; -} -static void cache_query_obj_on_fail(enum e_future_error err, const char * what, void * user) -{ - struct promise * p = (struct promise *) user; - struct cache_query_context* ctx=(struct cache_query_context*)promise_dettach_ctx(p); - promise_failed(p, err, what); - promise_finish(p); - ATOMIC_DEC(&(ctx->ref_handle->stat_val[STAT_CACHE_READING])); - cache_query_ctx_free_cb(ctx); - return; -} - - -struct cache_pending_context -{ - enum cache_pending_result status; - int is_undefined_obj; - char* req_if_none_match, *req_if_modified_since; - const struct tfe_http_half * request; - - char* url; - struct cached_meta cached_obj_meta; - enum OBJECT_LOCATION location; - struct cache_handle* ref_handle; - struct tango_cache_result* ref_tango_cache_result; - struct future* f_tango_cache_fetch; -}; -void cache_pending_ctx_free_cb(void* p) -{ - struct cache_pending_context* ctx=(struct cache_pending_context*)p; - ctx->request=NULL; - FREE(&(ctx->url)); - FREE(&(ctx->req_if_modified_since)); - FREE(&(ctx->req_if_none_match)); - if(ctx->f_tango_cache_fetch) - { - future_destroy(ctx->f_tango_cache_fetch); - } - FREE(&(ctx->cached_obj_meta.etag)); - FREE(&(ctx->cached_obj_meta.last_modified)); - FREE(&(ctx->cached_obj_meta.content_type)); - free(ctx); - return; -} -const struct cached_meta* cache_pending_result_read_meta(future_result_t * result, struct cache_mid* mid) -{ - struct cache_pending_context* ctx=(struct cache_pending_context*)result; - mid->location=ctx->location; - if(ctx->status==PENDING_RESULT_MISS) - { - return NULL; - } - else - { - return &(ctx->cached_obj_meta); - } -} - -static void cache_read_meta_on_succ(future_result_t * result, void * user) -{ - struct promise * p = (struct promise *) user; - struct cache_pending_context* ctx=(struct cache_pending_context*)promise_get_ctx(p); - struct tango_cache_result* _result=tango_cache_read_result(result); - ctx->ref_tango_cache_result=_result; - - switch(_result->type) - { - case RESULT_TYPE_HEADER: - ctx->cached_obj_meta.content_length=_result->tlength; - cached_meta_set(&ctx->cached_obj_meta, RESULT_TYPE_HEADER, _result->data_frag, _result->size); - ctx->status=PENDING_RESULT_REVALIDATE; - break; - case RESULT_TYPE_USERTAG: - cached_meta_set(&ctx->cached_obj_meta, RESULT_TYPE_USERTAG, _result->data_frag, _result->size); - TFE_LOG_DEBUG(ctx->ref_handle->logger, "cache meta read hit: %s %s %s" - , ctx->url - , ctx->cached_obj_meta.last_modified ? ctx->cached_obj_meta.last_modified:"no_last_modify" - , ctx->cached_obj_meta.etag ? ctx->cached_obj_meta.etag:"no_etag"); - break; - case RESULT_TYPE_MISS: - ctx->status=PENDING_RESULT_MISS; - TFE_LOG_DEBUG(ctx->ref_handle->logger, "cache meta read miss: %s", ctx->url); - //NOT break intentionally. - case RESULT_TYPE_END: - //last call. - ctx->location=_result->location; - ATOMIC_DEC(&(ctx->ref_handle->stat_val[STAT_CACHE_PENDING])); - promise_dettach_ctx(p); - promise_success(p, ctx); - cache_pending_ctx_free_cb(ctx); - break; - default: - break; - } - -} -static void cache_read_meta_on_fail(enum e_future_error err, const char * what, void * user) -{ - struct promise * p = (struct promise *) user; - struct cache_pending_context* ctx=(struct cache_pending_context*)promise_dettach_ctx(p); - promise_failed(p, err, what); - ATOMIC_DEC(&(ctx->ref_handle->stat_val[STAT_CACHE_PENDING])); - cache_pending_ctx_free_cb(ctx); - return; -} - - -#define CACHE_ACTION_BYPASS 0x80 -enum cache_pending_result web_cache_async_pending(struct cache_handle* handle, unsigned int thread_id, - const struct tfe_http_half * request, struct cache_mid** mid, struct future* f_revalidate) -{ - enum cache_pending_result result=PENDING_RESULT_FOBIDDEN; - long long cache_policy; - struct cache_param* param=&(handle->default_cache_policy); - void *ex_data=NULL; - struct maat_state *scan_mid=maat_state_new(handle->ref_feather, thread_id); - int ret=0; - size_t n_hit_result=0; - const char* cookie=NULL; - struct cache_mid* _mid=ALLOC(struct cache_mid, 1); - *mid=_mid; - cookie=tfe_http_std_field_read(request, TFE_HTTP_COOKIE); - if(cookie) - { - _mid->has_cookie=1; - } - _mid->is_dyn_url=is_dynamic_url(request->req_spec.url); - if(handle->cache_policy_enabled) - { - ret=maat_scan_string(handle->ref_feather, handle->table_url_constraint, request->req_spec.url, - strlen(request->req_spec.url), &cache_policy, 1, &n_hit_result, scan_mid); - - - if(cookie && ret<MAAT_SCAN_HIT) - { - ret=maat_scan_string(handle->ref_feather, handle->table_cookie_constraint, cookie, strlen(cookie), - &cache_policy, 1, &n_hit_result, scan_mid); - } - maat_state_free(scan_mid); - scan_mid=NULL; - - if(ret==MAAT_SCAN_HIT) - { - ex_data=maat_plugin_table_get_ex_data(handle->ref_feather, handle->cache_param_idx, (const char *)&cache_policy, sizeof(cache_policy)); - if(ex_data!=NULL) - { - param=(struct cache_param*)ex_data; - _mid->is_using_exception_param=1; - _mid->param=param; - - if((unsigned char)param->action==CACHE_ACTION_BYPASS) - { - _mid->shall_bypass=1; - } - _mid->cfg_id=param->config_id; - if(param->key_descr.is_not_empty) - { - _mid->cache_key=get_cache_key(request, &(param->key_descr)); - } - TFE_LOG_DEBUG(handle->logger, "cache policy %llu matched: url=%s alt-key=%s", - param->config_id, - request->req_spec.url, - _mid->cache_key!=NULL?_mid->cache_key:"null"); - ATOMIC_INC(&(handle->stat_val[STAT_CACHE_POLICY_MATCH])); - } - } - if(_mid->shall_bypass || - (!param->force_caching && !param->cache_dyn_url && _mid->is_dyn_url && param->key_descr.qs_num==0) || - (!param->force_caching && !param->cache_cookied_cont && _mid->has_cookie)) - { - _mid->result=PENDING_RESULT_FOBIDDEN; - return _mid->result; - } - } - enum cache_pending_action get_action=tfe_cache_get_pending(request, &(_mid->req_fresshness)); - switch(get_action) - { - case UNDEFINED: - if(!handle->query_undefined_obj_enabled) - { - ATOMIC_INC(&(handle->stat_val[STAT_CACHE_PEND_FORBIDDEN])); - result=PENDING_RESULT_FOBIDDEN; - } - else - { - ATOMIC_INC(&(handle->stat_val[STAT_CACHE_OVERRIDE_READ])); - result=PENDING_RESULT_ALLOWED; - } - break; - case FORBIDDEN: - if(param->ignore_req_nocache || param->force_caching) - { - result=PENDING_RESULT_ALLOWED; - } - else - { - ATOMIC_INC(&(handle->stat_val[STAT_CACHE_PEND_FORBIDDEN])); - result=PENDING_RESULT_FOBIDDEN; - } - break; - case ALLOWED: - result=PENDING_RESULT_ALLOWED; - break; - default: - if(param->no_revalidate) - { - result=PENDING_RESULT_ALLOWED; - } - else - { - result=PENDING_RESULT_REVALIDATE; - } - break; - } - if(result!=PENDING_RESULT_REVALIDATE) - { - _mid->result=result; - return _mid->result; - } - - struct tango_cache_meta_get meta; - memset(&meta, 0, sizeof(meta)); - meta.url=_mid->cache_key!=NULL?_mid->cache_key:request->req_spec.url; - meta.get = _mid->req_fresshness; - - struct promise* p=future_to_promise(f_revalidate); - struct cache_pending_context* ctx=ALLOC(struct cache_pending_context, 1); - ctx->status=PENDING_RESULT_FOBIDDEN; - ctx->ref_handle=handle; - ctx->url=tfe_strdup(request->req_spec.url); - ctx->req_if_modified_since=tfe_strdup(tfe_http_std_field_read(request, TFE_HTTP_IF_MODIFIED_SINCE)); - ctx->req_if_none_match=tfe_strdup(tfe_http_std_field_read(request, TFE_HTTP_IF_NONE_MATCH)); - promise_set_ctx(p, ctx, cache_pending_ctx_free_cb); - - ATOMIC_INC(&(handle->stat_val[STAT_CACHE_PENDING])); - ctx->f_tango_cache_fetch=future_create("_cache_pend", cache_read_meta_on_succ, cache_read_meta_on_fail, p); - ret=tango_cache_head_object(handle->clients[thread_id], ctx->f_tango_cache_fetch, &meta); - if(ret<0) - { - promise_dettach_ctx(p); - cache_pending_ctx_free_cb(ctx); - _mid->result=PENDING_RESULT_FOBIDDEN; - return _mid->result; - } - _mid->result=PENDING_RESULT_REVALIDATE; - - return _mid->result; -} -int web_cache_async_read(struct cache_handle* handle, unsigned int thread_id, - const struct tfe_http_half * request, struct cache_mid** mid, struct future* f) -{ - struct cache_query_context* query_ctx=NULL; - struct promise* p=NULL; - struct cache_mid* _mid=*mid; - assert(_mid->result!=PENDING_RESULT_FOBIDDEN); - - if(ATOMIC_READ(&(handle->stat_val[STAT_CACHE_READING])) > ATOMIC_READ(&(handle->put_concurrency_max))) - { - ATOMIC_INC(&(handle->stat_val[STAT_CACHE_READ_THROTTLE])); - return -1; - } - - struct tango_cache_meta_get meta; - memset(&meta, 0, sizeof(meta)); - meta.url=_mid->cache_key?_mid->cache_key:request->req_spec.url; - meta.get=_mid->req_fresshness; - query_ctx=ALLOC(struct cache_query_context, 1); - query_ctx->ref_handle=handle; - query_ctx->ref_mid=_mid; - query_ctx->url=tfe_strdup(request->req_spec.url); - - p=future_to_promise(f); - promise_allow_many_successes(p); - promise_set_ctx(p, query_ctx, cache_query_ctx_free_cb); - - ATOMIC_INC(&(handle->stat_val[STAT_CACHE_READING])); - query_ctx->f_tango_cache_fetch=future_create("_cache_read", cache_query_obj_on_succ, cache_query_obj_on_fail, p); - int ret=tango_cache_fetch_object(handle->clients[thread_id], query_ctx->f_tango_cache_fetch, &meta, _mid->location); - if(ret<0) - { - cache_query_ctx_free_cb(query_ctx); - return -1; - } - return 0; -} -struct cache_write_future_ctx -{ - char* url; - char upload_path[TFE_PATH_MAX]; - time_t start; - struct future* f; - struct cache_handle* ref_handle; -}; -void cache_write_future_ctx_free(struct cache_write_future_ctx* ctx) -{ - FREE(&(ctx->url)); - future_destroy(ctx->f); - free(ctx); -} -static void wrap_cache_write_on_succ(future_result_t * result, void * user) -{ - struct cache_write_future_ctx* ctx=(struct cache_write_future_ctx*)user; - TFE_LOG_DEBUG(ctx->ref_handle->logger, "cache upload success: %s path: %s elapse: %ld", - ctx->url, ctx->upload_path, time(NULL)-ctx->start); - cache_write_future_ctx_free(ctx); -} -static void wrap_cache_write_on_fail(enum e_future_error err, const char * what, void * user) -{ - struct cache_write_future_ctx* ctx=(struct cache_write_future_ctx*)user; - TFE_LOG_DEBUG(ctx->ref_handle->logger, "cache upload failed: %s %s lapse: %ld", ctx->url, what, time(NULL)-ctx->start); - ATOMIC_INC(&(ctx->ref_handle->stat_val[STAT_CACHE_WRITE_ERR])); - cache_write_future_ctx_free(ctx); -} - -struct cache_write_context* web_cache_write_start(struct cache_handle* handle, unsigned int thread_id, - const struct tfe_http_session * session, struct cache_mid **mid) -{ - struct cache_write_context* write_ctx=NULL; - struct response_freshness resp_freshness; - enum cache_pending_action put_action; - struct tango_cache_ctx *tango_cache_write_ctx=NULL; - char cont_type_str[TFE_STRING_MAX]={0}, user_tag_str[TFE_STRING_MAX]={0}; - const char* content_type=NULL; - char *tmp=NULL; - int i=0, is_undefined_obj=0; - size_t content_len=0; - const struct cache_param* param=NULL; - struct cache_mid* _mid=*mid; - - if(_mid!=NULL && _mid->is_using_exception_param) - { - param=_mid->param; - } - else - { - param=&(handle->default_cache_policy); - } - if(session->resp->resp_spec.content_length) - { - sscanf(session->resp->resp_spec.content_length, "%lu", &content_len); - } - content_type=tfe_http_std_field_read(session->resp, TFE_HTTP_CONT_TYPE); - if(content_type!=NULL&& NULL!=strcasestr(content_type, "text/html")) - { - _mid->is_html=1; - } - put_action=tfe_cache_put_pending(session->resp, &resp_freshness); - switch(put_action){ - case FORBIDDEN: - if(!(param->ignore_res_nocache || param->force_caching)) - { - ATOMIC_INC(&(handle->stat_val[STAT_CACHE_WRITE_FORBIDEN])); - TFE_LOG_DEBUG(handle->logger, "cache write forbiden: %s", session->req->req_spec.url); - return NULL; - } - break; - case REVALIDATE: - case ALLOWED: - case UNDEFINED: - if(param->force_caching) - { - break; - } - else if(_mid->shall_bypass - || (param->max_cache_obj_size!=0 && content_len > param->max_cache_obj_size) - || (param->min_cache_obj_size > content_len) - || (!param->cache_cookied_cont && _mid->has_cookie) - || (!param->cache_html && _mid->is_html) - ) - { - ATOMIC_INC(&(handle->stat_val[STAT_CACHE_WRITE_FORBIDEN])); - TFE_LOG_DEBUG(handle->logger, "cache write forbiden: %s, bypass:%d, cont_len:%zu, has_cookie:%d, is_html:%d", - session->req->req_spec.url, - _mid->shall_bypass, - content_len, - _mid->has_cookie, - _mid->is_html); - ATOMIC_INC(&(handle->stat_val[STAT_CACHE_WRITE_BYPASS])); - return NULL; - } - break; - default: - assert(0); - break; - } - if(ATOMIC_READ(&(handle->stat_val[STAT_CACHE_WRITING])) > handle->get_concurrency_max) - { - ATOMIC_INC(&(handle->stat_val[STAT_CACHE_WRITE_THROTTLE])); - return NULL; - } - const char* key=NULL; - size_t key_len=0; - if(param->min_use>0) - { - if(_mid->cache_key) - { - key=_mid->cache_key; - key_len=strlen(_mid->cache_key); - } - else - { - key=session->req->req_spec.url; - key_len=strlen(session->req->req_spec.url); - } - _mid->use_cnt=counting_bloom_check(handle->cache_key_bloom[thread_id].bloom, key, key_len); - - if(_mid->use_cnt<param->min_use) - { - counting_bloom_add(handle->cache_key_bloom[thread_id].bloom, key, key_len); - return NULL; - } - } - ATOMIC_INC(&(handle->stat_val[STAT_CACHE_WRITING])); - - struct tango_cache_meta_put meta; - memset(&meta, 0, sizeof(meta)); - meta.url=_mid->cache_key?_mid->cache_key:session->req->req_spec.url; - i=0; - - snprintf(cont_type_str, sizeof(cont_type_str), "content-type:%s",session->resp->resp_spec.content_type); - meta.std_hdr[i]=cont_type_str; - i++; - const char* etag=tfe_http_std_field_read(session->resp, TFE_HTTP_ETAG); - const char* last_modified=tfe_http_std_field_read(session->resp, TFE_HTTP_LAST_MODIFIED); - tmp=user_tag_str; - if(etag) tmp+=snprintf(tmp, sizeof(user_tag_str)-(tmp-user_tag_str), "etag:%s\r\n", etag); - if(last_modified) tmp+=snprintf(tmp, sizeof(user_tag_str)-(tmp-user_tag_str), "Last-modified:%s\r\n", last_modified); - if(strlen(user_tag_str)>0) - { - meta.usertag=user_tag_str; - meta.usertag_len=strlen(user_tag_str)+1; - } - meta.put=resp_freshness; - meta.put.timeout=MAX(param->pinning_time_sec, resp_freshness.timeout); - - struct cache_write_future_ctx* future_ctx=ALLOC(struct cache_write_future_ctx, 1); - future_ctx->url=tfe_strdup(session->req->req_spec.url); - future_ctx->start=time(NULL); - future_ctx->ref_handle=handle; - future_ctx->f=future_create("_cache_wrt", wrap_cache_write_on_succ, wrap_cache_write_on_fail, future_ctx); - tango_cache_write_ctx=tango_cache_update_start(handle->clients[thread_id], future_ctx->f, &meta); - if(tango_cache_write_ctx==NULL)//exceed maximum cache memory size. - { - cache_write_future_ctx_free(future_ctx); - return NULL; - } - TFE_LOG_DEBUG(handle->logger, "cache upload allowed: %s", future_ctx->url); - if(is_undefined_obj) - { - ATOMIC_INC(&(handle->stat_val[STAT_CACHE_OVERRIDE_WRITE])); - FS_operate(handle->fs_handle,handle->fs_id[STAT_CACHE_OVERRIDE_WRITE_OBJ_SIZE], 0, FS_OP_SET, content_len/1024); - } - FS_operate(handle->fs_handle,handle->fs_id[STAT_CACHE_WRITE_OBJ_SIZE], 0, FS_OP_SET, content_len/1024); - write_ctx=ALLOC(struct cache_write_context, 1); - write_ctx->write_ctx=tango_cache_write_ctx; - write_ctx->ref_cache_handle=handle; - write_ctx->content_len=content_len; - write_ctx->uploaded_len=0; - write_ctx->future_ctx=future_ctx; - return write_ctx; - -} -void web_cache_write(struct cache_write_context* ctx, const unsigned char * body_frag, size_t frag_size) -{ - tango_cache_update_frag_data(ctx->write_ctx, (const char*)body_frag, frag_size); - ctx->uploaded_len+=frag_size; - ATOMIC_ADD(&(ctx->ref_cache_handle->stat_val[STAT_CACHE_WRITE_BYTES]), frag_size); - return; -} -int web_cache_write_end(struct cache_write_context* ctx) -{ - int tmp_ret=0, ret=0; - struct cache_write_future_ctx* future_ctx=ctx->future_ctx; - if(ctx->uploaded_len==ctx->content_len) - { - tmp_ret=tango_cache_update_end(ctx->write_ctx, future_ctx->upload_path, sizeof(future_ctx->upload_path)); - if(tmp_ret<0) - { - //upload too slow or storage server error; - TFE_LOG_DEBUG(ctx->ref_cache_handle->logger, "cache upload failed: %s",ctx->future_ctx->url); - cache_write_future_ctx_free(ctx->future_ctx); - ctx->future_ctx=NULL; - ATOMIC_INC(&(ctx->ref_cache_handle->stat_val[STAT_CACHE_WRITE_ERR])); - ret=-1; - } - else - { - ret=1; - } - } - else - { - tango_cache_update_cancel(ctx->write_ctx); - ATOMIC_INC(&(ctx->ref_cache_handle->stat_val[STAT_CACHE_WRITE_CANCEL])); - ret=-2; - } - ATOMIC_DEC(&(ctx->ref_cache_handle->stat_val[STAT_CACHE_WRITING])); - ctx->write_ctx = NULL; - ctx->ref_cache_handle = NULL; - free(ctx); - return ret; -} - diff --git a/resource/pangu/pangu_http.json b/resource/pangu/pangu_http.json index 3306bf5..ea680e5 100644 --- a/resource/pangu/pangu_http.json +++ b/resource/pangu/pangu_http.json @@ -68,7 +68,7 @@ "is_valid": "yes", "groups": [ { - "virtual_table":"ATTR_HTTP_HOST_VIRTUAL", + "virtual_table":"ATTR_SERVER_FQDN", "group_name":"http_fqdn", "group_id":102, "not_flag":0, diff --git a/resource/pangu/table_info.conf b/resource/pangu/table_info.conf index d01a013..75a7258 100644 --- a/resource/pangu/table_info.conf +++ b/resource/pangu/table_info.conf @@ -129,15 +129,13 @@ }, { "table_id": 10, - "table_name": "ATTR_HTTP_HOST_VIRTUAL", - "db_tables": ["ATTR_HTTP_HOST", "ATTR_SERVER_FQDN"], + "table_name": "ATTR_SERVER_FQDN", "table_type": "virtual", "physical_table": "TSG_OBJ_FQDN" }, { "table_id": 11, - "table_name": "ATTR_HTTP_HOST_CAT_VIRTUAL", - "db_tables": ["ATTR_HTTP_HOST_CAT", "ATTR_SERVER_FQDN_CAT"], + "table_name": "ATTR_SERVER_FQDN_CAT", "table_type": "virtual", "physical_table": "TSG_OBJ_FQDN_CAT" }, @@ -457,18 +455,6 @@ }, { "table_id":43, - "table_name":"ATTR_DOH_HOST", - "table_type":"virtual", - "physical_table": "TSG_OBJ_FQDN" - }, - { - "table_id":44, - "table_name":"ATTR_DOH_HOST_CAT", - "table_type":"virtual", - "physical_table": "TSG_OBJ_FQDN_CAT" - }, - { - "table_id":45, "table_name":"PXY_SSL_FINGERPRINT", "table_type":"plugin", "valid_column":4, @@ -478,7 +464,7 @@ } }, { - "table_id":46, + "table_id":44, "table_name":"PXY_PROFILE_RUN_SCRIPTS", "table_type":"plugin", "valid_column":4, @@ -489,7 +475,7 @@ } }, { - "table_id":47, + "table_id":45, "table_name":"PXY_PROFILE_TCP_OPTION", "table_type":"plugin", "valid_column":6, @@ -499,7 +485,7 @@ } }, { - "table_id":48, + "table_id":46, "table_name":"SERVICE_CHAINING_COMPILE", "table_type":"plugin", "valid_column":9, @@ -509,7 +495,7 @@ } }, { - "table_id": 49, + "table_id": 47, "table_name": "APP_ID_DICT", "table_type": "plugin", "valid_column": 19, @@ -520,25 +506,25 @@ } }, { - "table_id":50, + "table_id":48, "table_name":"ATTR_APP_ID", "table_type":"virtual", "physical_table": "APP_ID_DICT" }, { - "table_id":51, + "table_id":49, "table_name":"ATTR_SUBSCRIBER_ID", "table_type":"virtual", "physical_table": "TSG_OBJ_SUBSCRIBER_ID" }, { - "table_id":52, + "table_id":50, "table_name":"ATTR_INTERNAL_ADDR", "table_type":"virtual", "physical_table": "TSG_OBJ_IP" }, { - "table_id":53, + "table_id":51, "table_name":"ATTR_EXTERNAL_ADDR", "table_type":"virtual", "physical_table": "TSG_OBJ_IP" diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index 6510cb2..00cbbd5 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -94,22 +94,6 @@ add_dependencies(nghttp2-static nghttp2) set_property(TARGET nghttp2-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libnghttp2.a) set_property(TARGET nghttp2-static APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include;${INSTALL_DIR}/src/nghttp2/lib) -### ratelimiter -ExternalProject_Add(ratelimiter PREFIX ratelimiter - URL ${CMAKE_CURRENT_SOURCE_DIR}/ratelimiter-1.1.0-x86_64.tar.gz - URL_MD5 6e4b9b31c70b9e84e2a6434fab6268f9 - CONFIGURE_COMMAND "" - BUILD_COMMAND make - BUILD_IN_SOURCE 1) - -ExternalProject_Get_Property(ratelimiter INSTALL_DIR) -file(MAKE_DIRECTORY ${INSTALL_DIR}/include) - -add_library(ratelimiter-static STATIC IMPORTED GLOBAL) -add_dependencies(ratelimiter-static ratelimiter) -set_property(TARGET ratelimiter-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libratelimiter.a) -set_property(TARGET ratelimiter-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) - #### GoogleTest ExternalProject_Add(googletest PREFIX googletest URL ${CMAKE_CURRENT_SOURCE_DIR}/googletest-release-1.8.0.tar.gz diff --git a/vendor/ratelimiter-1.1.0-x86_64.tar.gz b/vendor/ratelimiter-1.1.0-x86_64.tar.gz Binary files differdeleted file mode 100644 index 2a304ad..0000000 --- a/vendor/ratelimiter-1.1.0-x86_64.tar.gz +++ /dev/null |