diff options
| author | 彭宣正 <[email protected]> | 2021-05-26 11:10:59 +0800 |
|---|---|---|
| committer | 彭宣正 <[email protected]> | 2021-06-29 18:12:16 +0800 |
| commit | 8cb339003a3e7aeb0e85ee51ac1341900219ae86 (patch) | |
| tree | 16f98c8b14e0fb3fb4f4dd6adb9300d84b2c74d0 /example | |
| parent | 678bb1c6f93c1bb72924d2bf5d29d500e24de465 (diff) | |
✨ feat(src): 重构
Diffstat (limited to 'example')
| -rw-r--r-- | example/demo/CMakeLists.txt | 5 | ||||
| -rw-r--r-- | example/demo/hos_upload_complete.cpp | 147 | ||||
| -rw-r--r-- | example/demo/hos_write_complete.c | 158 | ||||
| -rwxr-xr-x | example/demo/test_size.sh | 16 | ||||
| -rwxr-xr-x | example/demo/test_times.sh | 15 | ||||
| -rw-r--r-- | example/performance/CMakeLists.txt | 2 | ||||
| -rw-r--r-- | example/performance/HosClientPerformance.cpp | 458 | ||||
| -rw-r--r-- | example/performance/conf/default.conf | 16 | ||||
| -rwxr-xr-x | example/performance/file/c.sh | 15 | ||||
| -rw-r--r-- | example/performance/file/file/1024k.data | bin | 1048576 -> 0 bytes | |||
| -rw-r--r-- | example/performance/file/file/1M.data | bin | 1048576 -> 0 bytes | |||
| -rw-r--r-- | example/performance/file/mail_192.168.50.43.58251>123.126.97.2.25_2396203646976.eml | 23 | ||||
| -rw-r--r-- | example/performance/file/test.txt | bin | 1048576 -> 0 bytes |
13 files changed, 207 insertions, 648 deletions
diff --git a/example/demo/CMakeLists.txt b/example/demo/CMakeLists.txt index 35c548c4..ec69aec1 100644 --- a/example/demo/CMakeLists.txt +++ b/example/demo/CMakeLists.txt @@ -1,16 +1,11 @@ cmake_minimum_required(VERSION 3.5) set(CMAKE_BUILD_TYPE Debug) project(hos_write_demo) -#project(hos_upload_complete) SET(CMAKE_BUILD_TYPE Debug) -#link_directories(/usr/local/lib64/) link_directories(/opt/MESA/lib/) -#link_libraries(hos-client-cpp) include_directories(/opt/MESA/include) add_executable(hos_write_demo hos_write_demo.cpp) -#add_executable(hos_upload_complete hos_upload_complete.cpp) target_link_libraries(hos_write_demo hos-client-cpp) -#target_link_libraries(hos_upload_complete hos-client-cpp) diff --git a/example/demo/hos_upload_complete.cpp b/example/demo/hos_upload_complete.cpp deleted file mode 100644 index 68740fb4..00000000 --- a/example/demo/hos_upload_complete.cpp +++ /dev/null @@ -1,147 +0,0 @@ -/************************************************************************* - > File Name: single_thread.cpp - > Author: pxz - > Created Time: Fri 11 Sep 2020 09:52:05 AM CST - ************************************************************************/ -extern "C" -{ -#include<stdio.h> -#include<stdlib.h> -#include<unistd.h> -#include<string.h> -#include<time.h> -} -#include"hos_client.h" - -//#define test_times 10000 - -#define debuginfo (void) - -typedef struct userdata_s -{ - struct timespec *finished; -}userdata_t; - -static size_t calc_time(struct timespec start, struct timespec end) -{ - return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec - - (start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec)); -} - -int file_to_buffer(const char *file, char *buffer, size_t *len) -{ - FILE *fp = fopen(file, "r"); - int num = 0; - *len = 0; - if (fp == NULL) - { - debuginfo("fopen file failed:%s\n", file); - return -1; - } - do{ - num = fread(&buffer[*len], 1, 4096, fp); - if (num < 0) - { - return -1; - } - *len += num; - }while(num == 4096); - fclose(fp); - return 0; -} - -void callback(bool result, const char *error, const char *bucket, const char *object, void *userdata) -{ - userdata_t *data = (userdata_t *)userdata; - clock_gettime(CLOCK_MONOTONIC, data->finished); - return ; -} - -int main(int argc, char *argv[]) -{ - if (argc != 4) - { - debuginfo("usege: singThread [bucket name] [object name]\n"); - return -1; - } - struct timespec start, end, finished; - size_t time; - int i = 0; - char *bucket = argv[1]; - char *object = argv[2]; - int test_times = atoi(argv[3]); - //int test_times = 10000; - //char *buf = (char *)malloc(1024 * 1024 * 4); - char buf[1024 * 1024 * 4]; - //char buf[1024 * 4]; - size_t buf_size; - int mode = FILE_MODE; - size_t fd[10000] = {0}; - userdata_t data = {&finished}; - - file_to_buffer(object, buf, &buf_size); - - debuginfo("hos_client_init start ...\n"); - hos_client_handle handle = hos_client_create("192.168.44.12", 9098, "default", "default", 3000); - if (handle == NULL) - { - debuginfo("error:hos_client_handle\n"); - return -1; - } - debuginfo("hos_client_init success ... \n"); - - debuginfo("hos_create_bucket start ... \n"); - if(hos_create_bucket(handle, bucket)) - { - debuginfo("hos_create_bucket failed ... \n"); - return -1; - } - debuginfo("hos_create_bucket success ... \n"); - - debuginfo("hos_verify_bucket start ... \n"); - if(!hos_verify_bucket(handle, bucket)) - { - debuginfo("hos_verify_bucket failed ... \n"); - return -1; - } - debuginfo("hos_verify_bucket success ... \n"); - -#if 1 - debuginfo("hos_upload_file start ...\n"); - clock_gettime(CLOCK_MONOTONIC, &start); - for (i = 0; i < test_times; i++) - { - hos_upload_file(handle, bucket, object, callback, (void *)&data, 0); - } - clock_gettime(CLOCK_MONOTONIC, &end); - time = calc_time(start, end); - time /= test_times; - printf("hos_upload_file spent %llu ns\n", time); - debuginfo("hos_upload_file end ...\n"); -#else - - debuginfo("hos_upload_buf start ...\n"); - clock_gettime(CLOCK_MONOTONIC, &start); - for (i = 0; i < test_times; i++) - { - hos_upload_buf(handle, bucket, object, buf, buf_len, callback, (void *)&data, 0); - } - clock_gettime(CLOCK_MONOTONIC, &end); - time = calc_time(start, end); - time /= test_times; - printf("hos_upload_buf spent %llu ns\n", time); - debuginfo("hos_upload_buf end ...\n"); - -#endif - debuginfo("hos_client_close start ...\n"); - if (hos_client_destory(handle) == 0) - { - time = calc_time(start, finished); - time /= test_times; - printf("hos upload finished spent %llu ns\n", time); - } - - debuginfo("hos_client_close end ...\n"); - - return 0; -} diff --git a/example/demo/hos_write_complete.c b/example/demo/hos_write_complete.c deleted file mode 100644 index e587c66d..00000000 --- a/example/demo/hos_write_complete.c +++ /dev/null @@ -1,158 +0,0 @@ -/************************************************************************* - > File Name: single_thread.cpp - > Author: pxz - > Created Time: Fri 11 Sep 2020 09:52:05 AM CST - ************************************************************************/ -#include<stdio.h> -#include<stdlib.h> -#include<unistd.h> -#include<string.h> -#include<time.h> -#include"hos_client.h" - -//#define test_times 10000 - -#define debuginfo (void) - -typedef struct userdata_s -{ - struct timespec *finished; -}userdata_t; - -static size_t calc_time(struct timespec start, struct timespec end) -{ - return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec - - (start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec)); -} - -int file_to_buffer(const char *file, char *buffer, size_t *len) -{ - FILE *fp = fopen(file, "r"); - int num = 0; - *len = 0; - if (fp == NULL) - { - debuginfo("fopen file failed:%s\n", file); - return -1; - } - do{ - num = fread(&buffer[*len], 1, 4096, fp); - if (num < 0) - { - return -1; - } - *len += num; - }while(num == 4096); - fclose(fp); - return 0; -} - -void callback(bool result, const char *error, const char *bucket, const char *object, void *userdata) -{ - userdata_t *data = (userdata_t *)userdata; - clock_gettime(CLOCK_MONOTONIC, data->finished); - return ; -} - -int main(int argc, char *argv[]) -{ - if (argc != 4) - { - debuginfo("usege: singThread [bucket name] [object name]\n"); - return -1; - } - struct timespec start, end, finished; - size_t time; - int i = 0; - char *bucket = argv[1]; - char *object = argv[2]; - int test_times = atoi(argv[3]); - char *buf = (char *)malloc(1024 * 1024 * 40); - size_t buf_size; - int mode = FILE_MODE; - size_t fd[10001] = {0}; - userdata_t data = {&finished}; - - file_to_buffer(object, buf, &buf_size); - - debuginfo("hos_client_init start ...\n"); - hos_client_handle handle = hos_client_create("192.168.40.223", 9098, "default", "default", 400); - //hos_client_handle handle = hos_client_create("http://192.168.32.10:9098/hos/", "default", "default", 4); - if (handle == NULL) - { - printf("error:hos_client_handle\n"); - return -1; - } - debuginfo("hos_client_init success ... \n"); - - debuginfo("hos_create_bucket start ... \n"); - if(hos_create_bucket(handle, bucket)) - { - printf("hos_create_bucket failed ... \n"); - return -1; - } - debuginfo("hos_create_bucket success ... \n"); - - debuginfo("hos_verify_bucket start ... \n"); - if(!hos_verify_bucket(handle, bucket)) - { - printf("hos_verify_bucket failed ... \n"); - return -1; - } - debuginfo("hos_verify_bucket success ... \n"); - -#if 1 - mode = FILE_MODE; - debuginfo("hos_upload_file start ...\n"); - clock_gettime(CLOCK_MONOTONIC, &start); - for (i = 0; i < test_times; i++) - { - fd[i] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode); - hos_write(fd[i], object, 0, 0); - } - clock_gettime(CLOCK_MONOTONIC, &end); - time = calc_time(start, end); - time /= test_times; - printf("hos_upload_file spent %llu ns\n", time); - debuginfo("hos_upload_file end ...\n"); -#else - - mode = BUFF_MODE | APPEND_MODE; -#if 0 - for (i = 0; i < test_times; i++) - { - } -#endif - debuginfo("hos_upload_buf start ...\n"); - clock_gettime(CLOCK_MONOTONIC, &start); - for (i = 0; i < test_times; i++) - { - fd[i] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode); - //printf("hos_write start...\n"); - if(hos_write(fd[i], buf, buf_size, 0, i + 1) != HOS_CLIENT_OK) - { - printf("error: hos_write!\n"); - } - //printf("hos_write success...\n"); - } - clock_gettime(CLOCK_MONOTONIC, &end); - time = calc_time(start, end); - time /= test_times; - printf("hos_upload_buf spent %llu ns\n", time); - debuginfo("hos_upload_buf end ...\n"); - -#endif - debuginfo("hos_client_close start ...\n"); - if (hos_client_destory(handle) == 0) - { - //printf("start:%lu\n", start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec); - //printf("finished:%lu\n", finished.tv_sec * 1000 * 1000 * 1000 + finished.tv_nsec); - time = calc_time(start, finished); - time /= test_times; - printf("hos upload finished spent %llu ns\n", time); - } - - debuginfo("hos_client_close end ...\n"); - free(buf); - return 0; -} diff --git a/example/demo/test_size.sh b/example/demo/test_size.sh deleted file mode 100755 index 3884091e..00000000 --- a/example/demo/test_size.sh +++ /dev/null @@ -1,16 +0,0 @@ -######################################################################### -# File Name: test_times.sh -# Author: pxz -# Created Time: Mon 21 Sep 2020 04:43:35 PM CST -######################################################################### -#!/bin/bash -test_size=("1k" "10k" "100k" "1M" "2M" "3M" "4M") -num=0 -echo ${test_size[${num}]} - -while((${num} < 7)) -do - ./hos_write_complete mybucket ./data/${test_size[$num]}.data 1000 - let "num++" -done - diff --git a/example/demo/test_times.sh b/example/demo/test_times.sh deleted file mode 100755 index 6e1adbc4..00000000 --- a/example/demo/test_times.sh +++ /dev/null @@ -1,15 +0,0 @@ -######################################################################### -# File Name: test_times.sh -# Author: pxz -# Created Time: Mon 21 Sep 2020 04:43:35 PM CST -######################################################################### -#!/bin/bash -test_times=(1, 10, 100, 1000, 10000) -num=0 - -while((${num} < 5)) -do - ./hos_write_complete mybucket my-file.txt ${test_times[$num]} - let "num++" -done - diff --git a/example/performance/CMakeLists.txt b/example/performance/CMakeLists.txt index 58d2c7cf..896a0633 100644 --- a/example/performance/CMakeLists.txt +++ b/example/performance/CMakeLists.txt @@ -27,5 +27,5 @@ endif() # end of for ASAN add_executable(HosClientPerformance HosClientPerformance.cpp) -target_link_libraries(HosClientPerformance hos-client-cpp) +target_link_libraries(HosClientPerformance hos-client-cpp MESA_handle_logger) diff --git a/example/performance/HosClientPerformance.cpp b/example/performance/HosClientPerformance.cpp index ac2a6175..6eb3b774 100644 --- a/example/performance/HosClientPerformance.cpp +++ b/example/performance/HosClientPerformance.cpp @@ -15,64 +15,39 @@ extern "C" #include<sys/stat.h> #include<math.h> #include<netinet/in.h> +#include<zlog.h> } #include"../../src/hos_client.h" +#include "MESA_handle_logger.h" #define MAX_THREAD_NUM 32 #ifndef MIN #define MIN(a,b) ((a) > (b)) ? (b) : (a) #endif -typedef struct conf_s -{ -#define STRING_SIZE 128 - char serverip[INET_ADDRSTRLEN]; - char bucket[STRING_SIZE]; - char object[STRING_SIZE]; - char file[STRING_SIZE]; - size_t port; - size_t pool_size; - size_t thread_sum; - size_t size; - float append_size; - size_t slice; - int mode; -}conf_t; +#define STRING_SIZE 1024 typedef struct thread_info_s { - conf_t conf; - hos_client_handle handle; + char object[STRING_SIZE]; + char bucket[STRING_SIZE]; + char file[100][STRING_SIZE]; + int mode; size_t thread_num; }thread_info_t; -static void configuration_init(conf_t *conf) -{ - strcpy(conf->serverip, "192.168.40.223"); - strcpy(conf->bucket, "mybucket"); - strcpy(conf->object, "myobject"); - strcpy(conf->file, "./file/test.txt"); - conf->port = 9098; - conf->pool_size = 4000; - conf->append_size = 1024; - conf->thread_sum = 1; - conf->mode = BUFF_MODE; - conf->slice = 0; - - return ; -} - -typedef struct userdata_s -{ - struct timespec *finished; -}userdata_t; +struct timespec *g_finished; +char g_file_name[100][STRING_SIZE]; +size_t g_mode; +size_t g_test_count; +size_t g_append_size; static size_t calc_time(struct timespec start, struct timespec end) { return (end.tv_sec - start.tv_sec) * 1000 * 1000 * 1000 + end.tv_nsec - start.tv_nsec; } -int read_file_list(const char *path, char file_name[][256]) +int read_file_list(const char *path, char file_name[][STRING_SIZE]) { DIR *dir; struct dirent *ptr; @@ -89,12 +64,12 @@ int read_file_list(const char *path, char file_name[][256]) { if(strcmp(ptr->d_name,".")==0 || strcmp(ptr->d_name,"..")==0) ///current dir OR parrent dir continue; - else if((ptr->d_type == 8) || (ptr->d_type == 10)) + else if((ptr->d_type == DT_REG) || (ptr->d_type == DT_LNK)) { memcpy(file_name[file_num], path, path_len); strcat(file_name[file_num], ptr->d_name); } - else if(ptr->d_type == 4) ///dir + else if(ptr->d_type == DT_DIR) ///dir { continue; } @@ -135,12 +110,11 @@ static int file_to_buffer(const char *file, char *buffer, size_t *len) return 0; } -static int upload_file(char *file, char *buff, int buff_len, int test_times, thread_info_t *thread_info, char *performance_info) +static int upload_file(char *file, char *buff, int buff_len, thread_info_t *thread_info, char *performance_info) { - int i; + size_t i; FILE *fp = NULL; size_t fd[3000]; - conf_t *conf = &thread_info->conf; struct timespec tstart, tend, twrite; long time_write = 0, time_upload = 0; size_t len = strlen(performance_info); @@ -149,11 +123,10 @@ static int upload_file(char *file, char *buff, int buff_len, int test_times, thr double variance = 0.00; double average = 0.00; long time = 0; - int success_cnt; //写文件 //clock_gettime(CLOCK_MONOTONIC, &tstart); - for (i = 0; i < test_times; i++) + for (i = 0; i < g_test_count; i++) { clock_gettime(CLOCK_MONOTONIC, &tstart); fp = fopen(file, "w+"); @@ -175,14 +148,14 @@ static int upload_file(char *file, char *buff, int buff_len, int test_times, thr } //clock_gettime(CLOCK_MONOTONIC, &twrite); //time_write = calc_time(tstart, twrite); - time_write /= test_times; + time_write /= g_test_count; //上传文件 //clock_gettime(CLOCK_MONOTONIC, &tstart); - for (i = 0; i < test_times; i++) + for (i = 0; i < g_test_count; i++) { clock_gettime(CLOCK_MONOTONIC, &tstart); - fd[i] = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode); + fd[i] = hos_open_fd(thread_info->bucket, thread_info->object, callback, NULL, thread_info->thread_num, g_mode); if (hos_write(fd[i], file, 0, thread_info->thread_num) != HOS_CLIENT_OK) { printf("error:hos_write file:%s\n", file); @@ -195,14 +168,14 @@ static int upload_file(char *file, char *buff, int buff_len, int test_times, thr } //clock_gettime(CLOCK_MONOTONIC, &tend); //time_upload = calc_time(tstart, tend); - time_upload /= test_times; + time_upload /= g_test_count; average = time_write + time_upload; - for (i = 0; i < test_times; i++) + for (i = 0; i < g_test_count; i++) { variance += pow((record[i] - average), 2); } - variance /= test_times; + variance /= g_test_count; sprintf(file_size, "%dk", buff_len / 1024); sprintf(&performance_info[len], "%-20lu%-20s%-20ld%-20ld%-20lf%-20lf\n", @@ -211,143 +184,122 @@ static int upload_file(char *file, char *buff, int buff_len, int test_times, thr return 0; } -static int upload_buff(char * buff, int buff_len, int test_times, thread_info_t *thread_info, char *performance_info) +static int upload_buff(char * buff, int buff_len, thread_info_t *thread_info, char *performance_info) { - int i = 0; + size_t i = 0; int j = 0; size_t fd[1000] = {0}; size_t tmp = 0; size_t rest = 0; struct timespec tstart, ttmp; - size_t time = 0; size_t len; - conf_t *conf = &thread_info->conf; char file_size[128]; char append_size[128]; size_t success_cnt = 0; int ret = 0; - int drop = 0; double variance = 0.00; double average = 0.00; long record[30000] = {0}; - if (conf->slice) + if (g_mode & APPEND_MODE) { - fd[0] = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode); - for (i = 0; i < test_times; i++) + fd[0] = hos_open_fd(thread_info->bucket, thread_info->object, callback, NULL, thread_info->thread_num, g_mode); + + for (i = 0; i < g_test_count; i++) { - j = 0; clock_gettime(CLOCK_MONOTONIC, &tstart); - while(1) + j = 0; + while (1) { - tmp = j * conf->slice; + tmp = j * g_append_size; rest = buff_len - tmp; - if (rest < conf->slice) + if (rest < g_append_size) { hos_write(fd[0], &buff[tmp], rest, thread_info->thread_num); break; } - hos_write(fd[0], &buff[tmp], conf->slice, thread_info->thread_num); + hos_write(fd[0], &buff[tmp], g_append_size, thread_info->thread_num); j++; } clock_gettime(CLOCK_MONOTONIC, &ttmp); record[i] = calc_time(tstart, ttmp); average += record[i]; } - average /= test_times; + average /= g_test_count; - for (i = 0; i < test_times; i++) + for (i = 0; i < g_test_count; i++) { variance += pow((record[i] - average), 2); } - variance /= test_times; - - sprintf(file_size, "%dk", buff_len / 1024); - sprintf(append_size, "%gk", conf->append_size / 1024); - len = strlen(performance_info); - sprintf(&performance_info[len], "%-20lu%-20s%-20s%-20d%-20lf%-20lf\n", - thread_info->thread_num, file_size, append_size, 0, average, sqrt(variance)); - }else{ + variance /= g_test_count; - if (conf->mode & APPEND_MODE) + if (buff_len > 1024 * 1024) { - fd[0] = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode); -#if 1 - for (i = 0; i < test_times; i++) - { - clock_gettime(CLOCK_MONOTONIC, &tstart); -#endif - j = 0; - while(1) - { - tmp = j * conf->append_size; - rest = buff_len - tmp; - if (rest < conf->append_size) - { - hos_write(fd[0], &buff[tmp], rest, thread_info->thread_num); - break; - } - hos_write(fd[0], &buff[tmp], conf->append_size, thread_info->thread_num); - j++; - } -#if 1 - clock_gettime(CLOCK_MONOTONIC, &ttmp); - record[i] = calc_time(tstart, ttmp); - average += record[i]; - } - average /= test_times; -#endif - - for (i = 0; i < test_times; i++) - { - variance += pow((record[i] - average), 2); - } - variance /= test_times; - - sprintf(file_size, "%dk", buff_len / 1024); - sprintf(append_size, "%gk", conf->append_size / 1024); - len = strlen(performance_info); - sprintf(&performance_info[len], "%-20lu%-20s%-20s%-20d%-20lf%-20lf\n", - thread_info->thread_num, file_size, append_size, 0, average,sqrt(variance)); - }else + sprintf(file_size, "%gM", (double)buff_len / 1024 / 1024); + } + else if (buff_len > 1024) + { + sprintf(file_size, "%gK", (double)buff_len / 1024); + } + else { - for (i = 0; i < test_times; i++) + sprintf(file_size, "%dB", buff_len); + } + sprintf(append_size, "%gK", (double)g_append_size / 1024); + len = strlen(performance_info); + sprintf(&performance_info[len], "%-20lu%-20s%-20s%-20lu%-20lf%-20lf\n", + thread_info->thread_num, file_size, append_size, g_test_count, average, sqrt(variance)); + } + else + { + for (i = 0; i < g_test_count; i++) + { + clock_gettime(CLOCK_MONOTONIC, &tstart); + fd[i] = hos_open_fd(thread_info->bucket, thread_info->object, callback, NULL, thread_info->thread_num, g_mode); + ret = hos_write(fd[i], buff, buff_len, thread_info->thread_num); + if (ret == HOS_CLIENT_OK) { - clock_gettime(CLOCK_MONOTONIC, &tstart); - fd[i] = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode); - ret = hos_write(fd[i], buff, buff_len, thread_info->thread_num); - if (ret == HOS_CLIENT_OK) - { - success_cnt++; - }else - { - //printf("error code:%d, thread_id:%d\n", ret, thread_info->thread_num); - //break; - } - clock_gettime(CLOCK_MONOTONIC, &ttmp); - record[i] = calc_time(tstart, ttmp); - average += record[i]; + success_cnt++; } - if (success_cnt) - average /= success_cnt; else - average /= test_times; - - for (i = 0; i < test_times; i++) { - variance += pow((record[i] - average), 2); + //printf("error code:%d, thread_id:%d\n", ret, thread_info->thread_num); + //break; } - variance /= test_times; + clock_gettime(CLOCK_MONOTONIC, &ttmp); + record[i] = calc_time(tstart, ttmp); + average += record[i]; + } + if (success_cnt) + average /= success_cnt; + else + average /= g_test_count; + + for (i = 0; i < g_test_count; i++) + { + variance += pow((record[i] - average), 2); + } + variance /= g_test_count; - sprintf(file_size, "%dk", buff_len / 1024); - sprintf(append_size, "%luk", conf->append_size / 1024); - len = strlen(performance_info); - sprintf(&performance_info[len], "%-20lu%-20s%-20d%-20d%-20lf%-20lf\n", - thread_info->thread_num, file_size, 0, 0, average, sqrt(variance)); + if (buff_len > 1024 * 1024) + { + sprintf(file_size, "%gM", (double)buff_len / 1024 / 1024); + } + else if (buff_len > 1024) + { + sprintf(file_size, "%gK", (double)buff_len / 1024); } + else + { + sprintf(file_size, "%dB", buff_len); + } + sprintf(append_size, "%gK", (double)g_append_size / 1024); + len = strlen(performance_info); + sprintf(&performance_info[len], "%-20lu%-20s%-20d%-20lu%-20lf%-20lf\n", + thread_info->thread_num, file_size, 0, g_test_count, average, sqrt(variance)); } - for (i = 0; i < test_times; i++) + for (i = 0; i < g_test_count; i++) { if (fd[i] > 2) { @@ -362,15 +314,11 @@ static void *put_object_thread(void *ptr) { char *performance_info = NULL; thread_info_t *thread_info = (thread_info_t *)ptr; - conf_t *conf = &thread_info->conf; char file[128]; size_t buff_len; int ret; int i; char *buff = NULL; - char file_name[100][256]; - struct stat s_buf; - int test_times = 1000; buff = (char *)malloc(30 * 1024 * 1024); if (buff == NULL) @@ -387,49 +335,26 @@ static void *put_object_thread(void *ptr) pthread_exit(NULL); } memset(performance_info, 0, 10240); - memset(file_name, 0, 100 *256); - stat(conf->file, &s_buf); - if (S_ISDIR(s_buf.st_mode)) + for (i = 0; i < 100; i++) { - read_file_list(conf->file, file_name); - for (i = 0; i < 100; i++) - { - if (file_name[i][0] == '\0') - break; - ret = file_to_buffer(file_name[i], buff, &buff_len); - if (ret == -1) - { - free(buff); - free(performance_info); - pthread_exit(NULL); - } - if (conf->mode & BUFF_MODE) - { - upload_buff(buff, buff_len, test_times, thread_info, performance_info); - }else - { - sprintf(file, "./file/file_%lu_%d", thread_info->thread_num, i); - upload_file(file, buff, buff_len, test_times, thread_info, performance_info); - } - } - }else - { - ret = file_to_buffer(conf->file, buff, &buff_len); + if (g_file_name[i][0] == '\0') + break; + ret = file_to_buffer(g_file_name[i], buff, &buff_len); if (ret == -1) { free(buff); free(performance_info); pthread_exit(NULL); } - - if (conf->mode & BUFF_MODE) + if (g_mode & BUFF_MODE) { - upload_buff(buff, buff_len, test_times, thread_info, performance_info); - }else + upload_buff(buff, buff_len, thread_info, performance_info); + } + else { - sprintf(file, "./file/file_%lu", thread_info->thread_num); - upload_file(file, buff, buff_len, test_times, thread_info, performance_info); + sprintf(file, "./file/file_%lu_%d", thread_info->thread_num, i); + upload_file(file, buff, buff_len, thread_info, performance_info); } } free(buff); @@ -440,92 +365,87 @@ int main(int argc, char *argv[]) { int ch; int buf_size; - conf_t conf; - char *object; char *retval; size_t thread_num; size_t thread[MAX_THREAD_NUM]; thread_info_t thread_info[MAX_THREAD_NUM]; cpu_set_t mask; FILE *log = NULL; - char log_name[256]; - const char *log_prefix = "./log/"; + char log_name[STRING_SIZE]; + const char *log_prefix = "./"; + char conf_path[STRING_SIZE] = {0}; + char bucket[STRING_SIZE] = {0}; + char object[STRING_SIZE] = {0}; + char module[STRING_SIZE] = {0}; + char upload_file_path[STRING_SIZE] = {0}; + size_t thread_sum = 0; time_t timep; -#if 0 - if (argc <= 1) - { - printf("usage: HosClientPerformance \n[-e set endpoint] \n[-b set bucket] \n" - "[-o set object] \n[-f set file] \n[-p set pool size] \n" - "[-t set thread sum] \n[-B set BUFF_MODE] \n" - "[-F set FILE_MODE] \n[-A set APPEND_MODE] \n" - "[-h show help info] \n"); - return -1; - } -#endif - configuration_init(&conf); + struct stat s_buf; + + /*init*/ + g_append_size = 102400; + memcpy(bucket, "hos_test_bucket", strlen("hos_test_bucket")); + memcpy(object, "object", strlen("object")); + memcpy(conf_path, "../conf/default.conf", strlen("../conf/default.conf")); + memcpy(module, "module", strlen("module")); + memcpy(upload_file_path, "../CMakeLists.txt", strlen("../CMakeLists.txt")); + thread_sum = 1; + g_mode = BUFF_MODE; + g_test_count = 100; + //读取命令行配置 - while((ch = getopt(argc, argv, "a:b:o:f:p:t:k:s:i:P:S:BFAh")) != -1) + while((ch = getopt(argc, argv, "a:b:c:o:m:f:t:M:n:h")) != -1) { switch(ch) { case 'a': - conf.append_size = 1024 * atof(optarg); + g_append_size = atoi(optarg); break; - case 'i': - //endpoint - buf_size = MIN(STRING_SIZE, strlen(optarg)); - strncpy((char *)conf.serverip, optarg, buf_size); - break; - case 'b': - buf_size = MIN(STRING_SIZE, strlen(optarg)); - strncpy((char *)conf.bucket, optarg, buf_size); - conf.bucket[buf_size] = '\0'; - break; - case 'o': + case 'b': /*bucket*/ buf_size = MIN(STRING_SIZE, strlen(optarg)); - strncpy(conf.object, optarg, buf_size); - conf.object[buf_size] = '\0'; + strncpy(bucket, optarg, buf_size); + bucket[buf_size] = '\0'; break; - case 'f': + case 'c': /*configuration file*/ buf_size = MIN(STRING_SIZE, strlen(optarg)); - strncpy(conf.file, optarg, buf_size); - conf.file[buf_size] = '\0'; - break; - case 'P': - conf.pool_size = atoi(optarg); - conf.pool_size = MIN(4000, conf.pool_size); - break; - case 't': - conf.thread_sum = atoi(optarg); + strncpy(conf_path, optarg, buf_size); + conf_path[buf_size] = '\0'; break; - case 'B': - conf.mode &= BUFF_MODE; - break; - case 'F': - conf.mode &= FILE_MODE; + case 'o': /*object*/ + buf_size = MIN(STRING_SIZE, strlen(optarg)); + strncpy(object, optarg, buf_size); + object[buf_size] = '\0'; break; - case 'A': - conf.mode |= APPEND_MODE; + case 'm': /*module*/ + buf_size = MIN(STRING_SIZE, strlen(optarg)); + strncpy(module, optarg, buf_size); + module[buf_size] = '\0'; break; - case 'k': - conf.append_size = 1024 * atoi(optarg); + case 'f': /*module*/ + buf_size = MIN(STRING_SIZE, strlen(optarg)); + strncpy(upload_file_path, optarg, buf_size); + upload_file_path[buf_size] = '\0'; break; - case 's': - conf.size = atoi(optarg); + case 't': + thread_sum = atoi(optarg); break; - case 'S': - conf.slice = atoi(optarg); + case 'M': + g_mode = atoi(optarg); break; - case 'p': - conf.port = atoi(optarg); + case 'n': + g_test_count = atoi(optarg); break; case 'h': default: - printf("usage: HosClientPerformance \n[-e set endpoint] \n[-b set bucket] \n" - "[-o set object] \n[-f set file] \n[-p set pool size] \n" - "[-t set thread sum] \n[-B set BUFF_MODE] \n" - "[-F set FILE_MODE] \n[-A set APPEND_MODE] \n" - "[-h show help info] \n[-a set append size(K)]\n"); + printf("usage: HosClientPerformance \n" + "[-b set bucket] \n" + "[-c set conf file path] \n" + "[-o set object] \n" + "[-t set thread sum] \n" + "[-m set module] \n" + "[-f set upload file path] \n" + "[-M set mode] \n" + "[-h show help info] \n"); return -1; break; } @@ -540,50 +460,52 @@ int main(int argc, char *argv[]) perror(log_name); return -1; } - //创建client - hos_client_handle handle = hos_client_create(conf.serverip, conf.port, "default", "default", conf.pool_size); - if (handle == NULL) + //初始化hos instance + hos_instance hos_instance = hos_init_instance(conf_path, module, thread_sum, bucket); + if (hos_instance == NULL) { - printf("error:hos_client_handle\n"); + printf("error:hos_client_handle\n %s\n", hos_instance->error_message); fclose(log); return -1; } - hos_set_thread_sum(handle, conf.thread_sum); - hos_set_cache_size(handle, conf.append_size); - hos_set_cache_count(handle, 0); - - //创建bucket - if (hos_create_bucket(handle, conf.bucket)) + //zlog_init("../conf/zlog.conf"); + MESA_handle_runtime_log_creation(NULL); + printf("\n==============================================================================================================================\n"); + if (g_mode & BUFF_MODE) { - printf("error:hos_create_bucket\n"); - fclose(log); - hos_client_destory(handle); - return -1; + printf("%-20s%-20s%-20s%-20s%-20s%-20s\n", "thread_id", "file_size", "append_size", "upload_time", "total_time", "std-dev"); + }else + { + printf("%-20s%-20s%-20s%-20s%-20s%-20s\n", "thread_id", "file_size", "write_time", "upload_time", "total_time", "std-dev"); } - printf("\n==============================================================================================================================\n"); - if (conf.mode & BUFF_MODE) + memset(g_file_name, 0, 100 * 256); + stat(upload_file_path, &s_buf); + if (S_ISDIR(s_buf.st_mode)) { - printf("%-20s%-20s%-20s%-20s%-20s\n", "thread_id", "file_size", "append_size", "upload_time", "total_time"); + read_file_list(upload_file_path, g_file_name); + for (int i = 0; i < 100; i++) + { + if (g_file_name[i][0] == '\0') + break; + } }else { - printf("%-20s%-20s%-20s%-20s%-20s\n", "thread_id", "file_size", "write_time", "upload_time", "total_time"); + memcpy(g_file_name[0], upload_file_path, MIN(strlen(upload_file_path), STRING_SIZE - 1)); } - hos_expand_fs2(handle, "./log/fs2.log", 0, "127.0.0.1", 8001); - //hos_expand_fs2(handle, NULL, 0, "127.0.0.1", 8001); - for ( thread_num = 0; thread_num < conf.thread_sum; thread_num++ ) + + for ( thread_num = 0; thread_num < thread_sum; thread_num++ ) { - thread_info[thread_num].conf = conf; - object = thread_info[thread_num].conf.object; - sprintf(&object[strlen(object)], "-%lu", thread_num); thread_info[thread_num].thread_num = thread_num; - thread_info[thread_num].handle = handle; + sprintf(thread_info[thread_num].object, "%s-%lu", object, thread_num); + sprintf(thread_info[thread_num].bucket, "%s", bucket); + if(pthread_create(&thread[thread_num], NULL, put_object_thread, (void *)&thread_info[thread_num])) { perror(" "); fclose(log); - hos_client_destory(handle); + hos_shutdown_instance(); return -1; } @@ -595,7 +517,7 @@ int main(int argc, char *argv[]) } } - for (thread_num = 0; thread_num < conf.thread_sum; thread_num++) + for (thread_num = 0; thread_num < thread_sum; thread_num++) { pthread_join(thread[thread_num], (void **)&retval); if (retval) @@ -606,7 +528,7 @@ int main(int argc, char *argv[]) } } - if (hos_client_destory(handle) == 0) + if (hos_shutdown_instance() == 0) { //time = calc_time(start, finished); //time /= test_times; diff --git a/example/performance/conf/default.conf b/example/performance/conf/default.conf new file mode 100644 index 00000000..9f80e557 --- /dev/null +++ b/example/performance/conf/default.conf @@ -0,0 +1,16 @@ +[module] +hos_serverip=192.168.44.12 +hos_serverport=9098 +hos_accesskeyid="default" +hos_secretkey="default" +hos_log_path="./hoslog" #default +hos_log_level=30 #default +hos_poolsize=100 #default +hos_thread_sum=32 #default +hos_cache_size=102400 #default +hos_cache_count=10 #default +hos_fd_live_time_ms=1000 #default +hos_fs2_serverip=127.0.0.1 +hos_fs2_serverport=10086 +hos_fs2_path="./hos_fs2_log" #default +hos_fs2_format=0 #default
\ No newline at end of file diff --git a/example/performance/file/c.sh b/example/performance/file/c.sh deleted file mode 100755 index b98adf7d..00000000 --- a/example/performance/file/c.sh +++ /dev/null @@ -1,15 +0,0 @@ -######################################################################### -# File Name: c.sh -# Author: pxz -# Created Time: Tue 13 Oct 2020 06:28:57 PM CST -######################################################################### -#!/bin/bash -size=("100" "200" "300" "400" "500" "600" "700" "800" "900" "1k" "1500" "10k" "100k" "1M" "2M" "3M" "4M" "5M" "10M" "20M" "30M") - -num=0 - -while((${num} < 11)) -do - truncate -s ${size[$num]} ${size[$num]}.data - let "num++" -done diff --git a/example/performance/file/file/1024k.data b/example/performance/file/file/1024k.data Binary files differdeleted file mode 100644 index 9e0f96a2..00000000 --- a/example/performance/file/file/1024k.data +++ /dev/null diff --git a/example/performance/file/file/1M.data b/example/performance/file/file/1M.data Binary files differdeleted file mode 100644 index 9e0f96a2..00000000 --- a/example/performance/file/file/1M.data +++ /dev/null diff --git a/example/performance/file/mail_192.168.50.43.58251>123.126.97.2.25_2396203646976.eml b/example/performance/file/mail_192.168.50.43.58251>123.126.97.2.25_2396203646976.eml deleted file mode 100644 index 044b96fb..00000000 --- a/example/performance/file/mail_192.168.50.43.58251>123.126.97.2.25_2396203646976.eml +++ /dev/null @@ -1,23 +0,0 @@ -Content-Type: multipart/mixed; boundary="===============1952501954711757490=="
-MIME-Version: 1.0
-From: [email protected]
-To: [email protected]
-Cc: [email protected]
-Subject: =?utf-8?b?5pyx5piO5piOc3ViamVjdA==?=
-
---===============1952501954711757490==
-Content-Type: text/plain; charset="utf-8"
-MIME-Version: 1.0
-Content-Transfer-Encoding: base64
-
-5pyx5piO5piOY29udGVudA==
-
---===============1952501954711757490==
-Content-Type: text/plain; charset="utf-8"
-MIME-Version: 1.0
-Content-Transfer-Encoding: base64
-Content-Disposition: attachment; filename="mail_attachment_name.txt"
-
-Y2VzaGk=
-
---===============1952501954711757490==--
diff --git a/example/performance/file/test.txt b/example/performance/file/test.txt Binary files differdeleted file mode 100644 index 93f84a69..00000000 --- a/example/performance/file/test.txt +++ /dev/null |
