From ae77019201515e1c94c8ca7c3ebab99c059ec641 Mon Sep 17 00:00:00 2001 From: lishu Date: Tue, 11 Dec 2018 23:09:04 +0800 Subject: æ›´æ–°cache为最新 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Makefile | 8 ++--- src/digest_detection.c | 4 +-- src/inc/cache_evbase_client.h | 33 +++++++++++-------- src/inc/tango_cache_client.h | 74 ++++++++++++++++++++++++++++--------------- src/main.c | 9 +++++- 5 files changed, 82 insertions(+), 46 deletions(-) diff --git a/src/Makefile b/src/Makefile index 09040ff..0fd2120 100644 --- a/src/Makefile +++ b/src/Makefile @@ -17,10 +17,10 @@ CC = g++ CCC = g++ LIB = -L./lib/ -LIB += -lssl -lcrypto -lxml2 -lcurl -levent -lpthread -LIB += -lMESA_handle_logger -lMESA_prof_load -lmaatframe -lrulescan -lMESA_htable -lMESA_field_stat2 -lWiredLB +#LIB += -lssl -lcrypto -lxml2 -lcurl -levent -lpthread +LIB += -lMESA_handle_logger -lMESA_prof_load -lmaatframe -lrulescan -lMESA_htable -lMESA_field_stat2 -lWiredLB -lpthread LIB += ./lib/libmy_socket.a -LIB += ./lib/pangu_tango_cache.a +LIB += ./lib/libtango_cache_client.a ./lib/libcjson.a ./lib/libcrypto.a ./lib/libcurl.a ./lib/libevent.a ./lib/libhiredis.a ./lib/libssl.a ./lib/libxml2.a -lz #LIB += ./lib/libdictator_debug.a LIB_FILE = $(wildcard ../lib/*.a) @@ -41,7 +41,7 @@ $(TARGET):$(OBJECTS) $(OBJECTSCPP) $(LIB_FILE) #$(CCC) -shared $(CFLAGS) $(OBJECTS) $(LIB) -o $@ $(CCC) $(CFLAGS) $(OBJECTS) $(OBJECTSCPP) $(LIB) $(LDDICTATOR) -o $@ cp $(TARGET) ../bin/ - cp $(TARGET) /home/mesasoft/dd/ +# cp $(TARGET) /home/mesasoft/dd/ .c.o: #%.d:%.c diff --git a/src/digest_detection.c b/src/digest_detection.c index 060bc23..bf3fcf0 100644 --- a/src/digest_detection.c +++ b/src/digest_detection.c @@ -165,7 +165,7 @@ void send_survey(frag_in_t* frg, media_t* mdi, struct Maat_rule_t* scan_result) char buf_survey[BUF_SIZE] = {0}; msg_head_t* mh_survey = (msg_head_t*)buf_survey; msg_result_t* msi_survey = (msg_result_t *)(buf_survey + sizeof(msg_head_t)); - struct tango_cache_meta meta; + struct tango_cache_meta_put meta; struct future_pdata* pdata; char key[256] = {0}; char out_fullfilename[256] = {0}; @@ -187,7 +187,7 @@ void send_survey(frag_in_t* frg, media_t* mdi, struct Maat_rule_t* scan_result) localtime_r(&tv.tv_sec, &now); strftime(day_time, sizeof(day_time), "%Y%m%d", &now); snprintf(key, sizeof(key), "%s/%lu.%s", day_time, mdi->mid, gen_filesuffix_by_mediatype(mdi->media_type)); - memset(&meta, 0, sizeof(struct tango_cache_meta)); + memset(&meta, 0, sizeof(struct tango_cache_meta_put)); meta.url = key; pdata = (struct future_pdata *)calloc(1, sizeof(struct future_pdata)); diff --git a/src/inc/cache_evbase_client.h b/src/inc/cache_evbase_client.h index 01bbca4..5bead2f 100644 --- a/src/inc/cache_evbase_client.h +++ b/src/inc/cache_evbase_client.h @@ -20,40 +20,47 @@ struct cache_evbase_ctx struct cache_evbase_instance *instance_asyn; }; -/*ËùÓÐAPIÏ̲߳»°²È«£¬APIµÄʹÓÃ˵Ã÷²Î¿¼tango_cache_client.h*/ +/*ËùÓÐAPIḬ̈߳²È«£¬APIµÄʹÓÃ˵Ã÷²Î¿¼tango_cache_client.h*/ enum CACHE_ERR_CODE cache_evbase_get_last_error(const struct cache_evbase_ctx *ctx_asyn); enum CACHE_ERR_CODE cache_evbase_ctx_error(const struct cache_evbase_instance *instance); void cache_evbase_get_statistics(const struct cache_evbase_instance *instance, struct cache_statistics *out); -/*´´½¨ÊµÀý£¬Ã¿Ïß³ÌÒ»¸ö£¬»òʹÓÃʱ¼ÓËø*/ -struct cache_evbase_instance *cache_evbase_instance_new(const char* profile_path, const char* section, void *runtimelog); +void cache_evbase_global_init(void); + +//ÿ¸öminio¼¯ÈººÍbucket´´½¨Ò»¸öparameter£¬¶à¸öinstance¿É¹²ÏíÒ»¸öparameter£» +struct tango_cache_parameter *cache_evbase_parameter_new(const char* profile_path, const char* section, void *runtimelog); + +/*´´½¨ÊµÀý£¬Ḭ̈߳²È«£¬ÄÚ²¿»áÆô¶¯Ò»¸öÏß³Ì*/ +struct cache_evbase_instance *cache_evbase_instance_new(struct tango_cache_parameter *param, void *runtimelog); //GET½Ó¿Ú£¬³É¹¦·µ»Ø0£¬Ê§°Ü·µ»Ø-1£¬future»Øµ÷º¯Êý»áÔÚÁíÍâµÄÏß³ÌÖÐÖ´ÐУ¬ÏÂͬ -int cache_evbase_fetch_object(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta); +int cache_evbase_fetch_object(struct cache_evbase_instance *instance, struct future* f, struct tango_cache_meta_get *meta); +int cache_evbase_head_object(struct cache_evbase_instance *instance, struct future* f, struct tango_cache_meta_get *meta); struct tango_cache_result *cache_evbase_read_result(void *promise_result); //DELETE½Ó¿Ú -int cache_evbase_delete_object(struct cache_evbase_instance *instance, struct future* future, const char *objkey); +int cache_evbase_delete_object(struct cache_evbase_instance *instance, struct future* f, const char *objkey); //Ò»´ÎÐÔÉÏ´«½Ó¿Ú -int cache_evbase_upload_once_data(struct cache_evbase_instance *instance, struct future* future, +int cache_evbase_upload_once_data(struct cache_evbase_instance *instance, struct future* f, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, - struct tango_cache_meta *meta, - char *path, size_t pathsize); -int cache_evbase_upload_once_evbuf(struct cache_evbase_instance *instance, struct future* future, + struct tango_cache_meta_put *meta, + char *path/*OUT*/, size_t pathsize); +int cache_evbase_upload_once_evbuf(struct cache_evbase_instance *instance, struct future* f, struct evbuffer *evbuf, - struct tango_cache_meta *meta, - char *path, size_t pathsize); + struct tango_cache_meta_put *meta, + char *path/*OUT*/, size_t pathsize); //Á÷ʽÉÏ´«½Ó¿Ú -struct cache_evbase_ctx *cache_evbase_update_start(struct cache_evbase_instance *instance, struct future* future, struct tango_cache_meta *meta); +struct cache_evbase_ctx *cache_evbase_update_start(struct cache_evbase_instance *instance, struct future* f, struct tango_cache_meta_put *meta); int cache_evbase_update_frag_data(struct cache_evbase_ctx *ctx_asyn, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size); int cache_evbase_update_frag_evbuf(struct cache_evbase_ctx *ctx_asyn, struct evbuffer *evbuf); void cache_evbase_update_end(struct cache_evbase_ctx *ctx_asyn); +void cache_evbase_update_cancel(struct cache_evbase_ctx *ctx_asyn); -void cache_evbase_get_object_path(const struct cache_evbase_ctx *ctx, char *path, size_t pathsize); +void cache_evbase_get_object_path(const struct cache_evbase_ctx *ctx, char *path/*OUT*/, size_t pathsize); #endif diff --git a/src/inc/tango_cache_client.h b/src/inc/tango_cache_client.h index bcd2f63..c551e8f 100644 --- a/src/inc/tango_cache_client.h +++ b/src/inc/tango_cache_client.h @@ -4,8 +4,8 @@ #include #include -#include "tfe_future.h" -#include "proxy_cache.h" +#include +#include "tango_cache_pending.h" #define USER_TAG_MAX_LEN 1518 @@ -16,7 +16,13 @@ enum CACHE_ERR_CODE CACHE_TIMEOUT, //»º´æ³¬Ê± CACHE_OUTOF_MEMORY,//µ±Ç°ÄÚ´æÕ¼Óó¬¹ýÏÞÖÆ£¬²é¿´MAX_USED_MEMORY_SIZE_MBÊÇ·ñ¹ýС»òÕßµ±Ç°ÉÏ´«ËÙÂʸú²»Éϵ÷ÓÃÕßµÄËÙÂÊ CACHE_ERR_CURL, - CACHE_ERR_UNKNOWN + CACHE_ERR_WIREDLB, + CACHE_ERR_SOCKPAIR, + CACHE_ERR_INTERNAL, + CACHE_ERR_REDIS_JSON, + CACHE_ERR_REDIS_CONNECT, + CACHE_OUTOF_SESSION, + CACHE_UPDATE_CANCELED, }; enum PUT_MEMORY_COPY_WAY @@ -41,23 +47,27 @@ struct cache_statistics long long put_error_num;//UPLOADʧ°ÜµÄ´ÎÊý long long del_recv_num; //·¢ÆðDELETEµÄ´ÎÊý long long del_succ_num; //DELETE³É¹¦µÄ´ÎÊý - long long del_error_num;//DELETE³É¹¦µÄ´ÎÊý + long long del_error_num;//DELETEʧ°ÜµÄ´ÎÊý + long long totaldrop_num;//ÄÚ´æÂúÒÔ¼°WiredLB³ö´íʱDROPµÄ´ÎÊý long long memory_used; //µ±Ç°UPLOAD BODYËùÕ¼ÄÚ´æ´óС long long session_num; //µ±Ç°ÕýÔÚ½øÐÐGET/PUTµÄHTTP»á»°Êý }; enum CACHE_RESULT_TYPE { - RESULT_TYPE_HEADER=0, //Ö»µ÷ÓÃÒ»´Î - RESULT_TYPE_USERTAG, //Ö»µ÷ÓÃÒ»´Î + RESULT_TYPE_HEADER=0, //×î¶àÖ»µ÷ÓÃÒ»´Î + RESULT_TYPE_USERTAG, //×î¶àÖ»µ÷ÓÃÒ»´Î RESULT_TYPE_BODY, //¿ÉÄܵ÷Óöà´Î + RESULT_TYPE_END, //È«²¿½áÊø£¬Ö»µ÷ÓÃÒ»´Î£¬²»¸½´øÊý¾Ý + RESULT_TYPE_MISS, //»º´æδÃüÖУ¬ÓëÆäËûÀàÐÍ»¥³â£¬Ö»µ÷ÓÃÒ»´Î(ÓÐENDÖ®Òâ)£¬²»¸½´øÊý¾Ý }; //promise_successµÄ½á¹ûresult struct tango_cache_result { - const void *data_frag; //Èç¹ûtypeΪRESULT_TYPE_HEADER£¬Ã¿¸öÍ·²¿ºó»á°üº¬Ò»¸ö»»ÐÐ(HTTP1.1¸ñʽ) + const char *data_frag; //Èç¹ûtypeΪRESULT_TYPE_HEADER£¬Ã¿¸öÍ·²¿ºó»á°üº¬Ò»¸ö»»ÐÐ(HTTP1.1¸ñʽ) size_t size; + size_t tlength; //¶ÔÏóµÄ×ܳ¤¶È£¬»Øµ÷ʱ¶¼ÓÐЧ enum CACHE_RESULT_TYPE type; }; @@ -71,18 +81,22 @@ enum CACHE_HTTP_HDR_TYPE HDR_CONTENT_NUM, }; -struct tango_cache_meta +struct tango_cache_meta_get { - const char* url; //»º´æ:URL£»·Ç½á¹¹»¯ÈÕÖ¾:ÎļþÃû£»×î´ó³¤¶È256×Ö½Ú + const char* url; //»º´æ:URL£»·Ç½á¹¹»¯ÈÕÖ¾:Îļþ·¾¶Ãû¡£CACHE_OBJECT_KEY_HASH_SWITCH=0ʱ×î´ó³¤¶È256×Ö½Ú£¬=1ʱÎÞÏÞÖÆ + struct request_freshness get; +}; + +struct tango_cache_meta_put +{ + const char* url; const char* std_hdr[HDR_CONTENT_NUM]; //ÍêÕûÍ·²¿£¬Èç"Content-Type: text/html"£¬²»Òª°üº¬»»ÐУ»NULL±íʾûÓиÃÍ·²¿£» const char* usertag; //¿ÉÒÔÊÇÈÎÒâÄÚÈÝ£¬GETʱ»áÔ­Ñù·µ»Ø size_t usertag_len; //×î´ó³¤¶ÈUSER_TAG_MAX_LEN£¬0±íʾûÓиÃÍ·²¿ - union{ - struct response_freshness put; - struct request_freshness get; - }; + struct response_freshness put; }; +struct tango_cache_parameter; struct tango_cache_instance; struct tango_cache_ctx; @@ -90,22 +104,28 @@ enum CACHE_ERR_CODE tango_cache_get_last_error(const struct tango_cache_ctx *ctx enum CACHE_ERR_CODE tango_cache_ctx_error(const struct tango_cache_instance *instance); void tango_cache_get_statistics(const struct tango_cache_instance *instance, struct cache_statistics *out); +/*ÿ¸ö½ø³ÌÖ´ÐÐÒ»´Î³õʼ»¯*/ +void tango_cache_global_init(void); + +//ÿ¸öminio¼¯ÈººÍbucket´´½¨Ò»¸öparameter£¬¶à¸öinstance¿É¹²ÏíÒ»¸öparameter +struct tango_cache_parameter *tango_cache_parameter_new(const char* profile_path, const char* section, void *runtimelog); /*ÒÔÏÂËùÓÐAPIÏ̲߳»°²È«*/ //ÿ¸ö¼àÌýÏ̴߳´½¨Ò»¸öinstance -struct tango_cache_instance *tango_cache_instance_new(struct event_base* evbase,const char* profile_path, const char* section, void *runtimelog); +struct tango_cache_instance *tango_cache_instance_new(struct tango_cache_parameter *param, struct event_base* evbase, void *runtimelog); /* GET½Ó¿ÚµÄAPI*/ -//³É¹¦Ê±»Øµ÷promise_success, resultΪNULLʱ±íʾ½áÊø£» +//³É¹¦Ê±»Øµ÷promise_success //ʧ°Üʱ»Øµ÷promise_failed(½öÒ»´Î)£¬Ê¹ÓÃget_last_error»ñÈ¡´íÎóÂ룻 //future²»¿ÉÒÔΪNULL -int tango_cache_fetch_object(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta); +int tango_cache_fetch_object(struct tango_cache_instance *instance, struct future* f, struct tango_cache_meta_get *meta); +int tango_cache_head_object(struct tango_cache_instance *instance, struct future* f, struct tango_cache_meta_get *meta); //´Ópromise_successµÄresult²ÎÊýÌáÈ¡½á¹û -struct tango_cache_result *tango_cache_read_result(void *promise_result); +struct tango_cache_result *tango_cache_read_result(future_result_t *promise_result); /* DELETE½Ó¿ÚµÄAPI*/ -int tango_cache_delete_object(struct tango_cache_instance *instance, struct future* future, const char *objkey); +int tango_cache_delete_object(struct tango_cache_instance *instance, struct future* f, const char *objkey); /* UPLOAD½Ó¿ÚµÄAPI @@ -115,24 +135,26 @@ int tango_cache_delete_object(struct tango_cache_instance *instance, struct futu /*ÍêÕûÒ»´ÎÉÏ´«API*/ //Èôpath²»Îª¿Õ£¬ÔòÊä³ö¶ÔÏóµÄ´æ´¢Â·¾¶ //·µ»ØÖµ: 0-³É¹¦£»<0ʧ°Ü£»ÏÂͬ -int tango_cache_upload_once_data(struct tango_cache_instance *instance, struct future* future, +int tango_cache_upload_once_data(struct tango_cache_instance *instance, struct future* f, enum PUT_MEMORY_COPY_WAY way, const char *data, size_t size, - struct tango_cache_meta *meta, - char *path, size_t pathsize); -int tango_cache_upload_once_evbuf(struct tango_cache_instance *instance, struct future* future, + struct tango_cache_meta_put *meta, + char *path/*OUT*/, size_t pathsize); +int tango_cache_upload_once_evbuf(struct tango_cache_instance *instance, struct future* f, enum EVBUFFER_COPY_WAY way, struct evbuffer *evbuf, - struct tango_cache_meta *meta, - char *path, size_t pathsize); + struct tango_cache_meta_put *meta, + char *path/*OUT*/, size_t pathsize); /*Á÷ʽÉÏ´«API*/ //·µ»ØÖµ: ÈôΪNULLÔò±íʾ´´½¨Ê§°Ü£¬µ÷ÓÃtango_cache_ctx_error²é¿´´íÎóÂëÊÇ·ñÊÇCACHE_OUTOF_MEMORY(Õý³£Çé¿öÏÂÊÇ)£» -struct tango_cache_ctx *tango_cache_update_start(struct tango_cache_instance *instance, struct future* future, struct tango_cache_meta *meta); +struct tango_cache_ctx *tango_cache_update_start(struct tango_cache_instance *instance, struct future* f, struct tango_cache_meta_put *meta); //·µ»ØÖµ: 0-³É¹¦£»<0ʧ°Ü£¬µ÷ÓÃtango_cache_get_last_error²é¿´´íÎóÂ룻 int tango_cache_update_frag_data(struct tango_cache_ctx *ctx, const char *data, size_t size); int tango_cache_update_frag_evbuf(struct tango_cache_ctx *ctx, enum EVBUFFER_COPY_WAY way, struct evbuffer *evbuf); void tango_cache_update_end(struct tango_cache_ctx *ctx); +//µ÷ÓÃcancelºó²»»á²úÉú½á¹û»Øµ÷£¬ÈÏ×÷´«Êäʧ°Ü +void tango_cache_update_cancel(struct tango_cache_ctx *ctx); //»ñÈ¡¶ÔÏókeyÖµ£»µ±CACHE_OBJECT_KEY_HASH_SWITCH=1¿ªÆô¶ÔURL/ÎļþÃû¹þϣʱÓÐÓà -void tango_cache_get_object_path(const struct tango_cache_ctx *ctx, char *path, size_t pathsize); +void tango_cache_get_object_path(const struct tango_cache_ctx *ctx, char *path/*OUT*/, size_t pathsize); #endif diff --git a/src/main.c b/src/main.c index 3f0500b..facf520 100644 --- a/src/main.c +++ b/src/main.c @@ -248,16 +248,23 @@ int main(int argc, char **argv) memset(&g_dd_run, 0, sizeof(dd_parameter_t)); memset(&g_dd_cfg, 0, sizeof(dd_configure_t)); memset(&g_dd_stat, 0, sizeof(dd_status_t)); + struct tango_cache_parameter *parameter; /*read main.conf and init*/ if(-1==read_conf_and_init("./conf/main.conf")) { return -1; } + + parameter = cache_evbase_parameter_new("./conf/main.conf", "TANGO_CACHE", g_dd_run.logger); + if(parameter == NULL) + { + return -1; + } for(uint32_t i=0;i