summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorpengxuanzheng <[email protected]>2020-10-14 15:23:01 +0800
committerpengxuanzheng <[email protected]>2020-11-02 19:00:02 +0800
commit3f38cefc5ee10e0e0814a4ee4b94cfe1e0b5a34b (patch)
treed922d1f3fe63e7273aca1721bc73ef79e25279b0
parent8a35250d2127feabf66fb536838043a13a32f7d1 (diff)
增加 performan 测试
-rwxr-xr-xexample/data/test_size.sh18
-rw-r--r--example/demo/CMakeLists.txt (renamed from example/CMakeLists.txt)0
-rw-r--r--example/demo/data/100k.data (renamed from example/data/100k.data)bin102400 -> 102400 bytes
-rw-r--r--example/demo/data/10M.databin0 -> 10485760 bytes
-rw-r--r--example/demo/data/10k.data (renamed from example/data/10k.data)bin10240 -> 10240 bytes
-rw-r--r--example/demo/data/11k.databin0 -> 11264 bytes
-rw-r--r--example/demo/data/12k.databin0 -> 12288 bytes
-rw-r--r--example/demo/data/13k.databin0 -> 13312 bytes
-rw-r--r--example/demo/data/14k.databin0 -> 14336 bytes
-rw-r--r--example/demo/data/15k.databin0 -> 15360 bytes
-rw-r--r--example/demo/data/1M.data (renamed from example/data/1M.data)bin1048576 -> 1048576 bytes
-rw-r--r--example/demo/data/1k.data (renamed from example/data/1k.data)bin1024 -> 1024 bytes
-rw-r--r--example/demo/data/20M.databin0 -> 20971520 bytes
-rw-r--r--example/demo/data/2M.data (renamed from example/data/2M.data)bin2097152 -> 2097152 bytes
-rw-r--r--example/demo/data/2k.databin0 -> 2048 bytes
-rw-r--r--example/demo/data/30M.databin0 -> 31457280 bytes
-rw-r--r--example/demo/data/3M.data (renamed from example/data/3M.data)bin3145728 -> 3145728 bytes
-rw-r--r--example/demo/data/3k.databin0 -> 3072 bytes
-rw-r--r--example/demo/data/4M.data (renamed from example/data/4M.data)bin4194304 -> 4194304 bytes
-rw-r--r--example/demo/data/4k.databin0 -> 4096 bytes
-rw-r--r--example/demo/data/5M.databin0 -> 5242880 bytes
-rw-r--r--example/demo/data/5k.databin0 -> 5120 bytes
-rw-r--r--example/demo/data/6k.databin0 -> 6144 bytes
-rw-r--r--example/demo/data/7k.databin0 -> 7168 bytes
-rw-r--r--example/demo/data/8k.databin0 -> 8192 bytes
-rw-r--r--example/demo/data/9k.databin0 -> 9216 bytes
-rw-r--r--example/demo/data/append.txt1
-rw-r--r--example/demo/data/hello.txt1
-rwxr-xr-xexample/demo/data/hos_upload_completebin0 -> 17208 bytes
-rwxr-xr-xexample/demo/data/hos_write_completebin0 -> 17360 bytes
-rwxr-xr-xexample/demo/data/test_size.sh20
-rwxr-xr-xexample/demo/data/test_times.sh16
-rw-r--r--example/demo/hos_upload_complete.cpp (renamed from example/hos_upload_complete.cpp)4
-rw-r--r--example/demo/hos_write_complete.cpp (renamed from example/hos_write_complete.cpp)38
-rwxr-xr-xexample/demo/test_size.sh (renamed from example/test_size.sh)0
-rwxr-xr-xexample/demo/test_times.sh (renamed from example/test_times.sh)0
-rw-r--r--example/performance/CMakeLists.txt13
-rw-r--r--example/performance/HosClientPerformance.cpp542
-rw-r--r--example/performance/file/100k.databin0 -> 102400 bytes
-rw-r--r--example/performance/file/10M.databin0 -> 10485760 bytes
-rw-r--r--example/performance/file/10M.txtbin0 -> 10485760 bytes
-rw-r--r--example/performance/file/10k.databin0 -> 10240 bytes
-rw-r--r--example/performance/file/1M.databin0 -> 1048576 bytes
-rw-r--r--example/performance/file/1k.databin0 -> 1024 bytes
-rw-r--r--example/performance/file/20M.databin0 -> 20971520 bytes
-rw-r--r--example/performance/file/2M.databin0 -> 2097152 bytes
-rw-r--r--example/performance/file/30M.databin0 -> 31457280 bytes
-rw-r--r--example/performance/file/3M.databin0 -> 3145728 bytes
-rw-r--r--example/performance/file/4M.databin0 -> 4194304 bytes
-rw-r--r--example/performance/file/5M.databin0 -> 5242880 bytes
-rwxr-xr-xexample/performance/file/c.sh15
-rw-r--r--example/performance/file/file/1024k.databin0 -> 1048576 bytes
-rw-r--r--example/performance/file/file/1M.databin0 -> 1048576 bytes
-rw-r--r--example/performance/file/file_0bin0 -> 1024 bytes
-rw-r--r--example/performance/file/test.txtbin0 -> 1048576 bytes
-rwxr-xr-xexample/performance/test.sh17
-rw-r--r--src/hos_client.cpp11
-rw-r--r--src/hos_client.h4
58 files changed, 659 insertions, 41 deletions
diff --git a/example/data/test_size.sh b/example/data/test_size.sh
deleted file mode 100755
index 8fbb5533..00000000
--- a/example/data/test_size.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#########################################################################
-# File Name: test_times.sh
-# Author: pxz
-# Created Time: Mon 21 Sep 2020 04:43:35 PM CST
-#########################################################################
-#!/bin/bash
-test_size=("1k" "10k" "100k" "1M" "2M" "3M" "4M" "5M" "10M" "20M" "30M")
-num=0
-
-while((${num} < 11))
-do
- #echo ./hos_write_complete mybucket ${test_size[$num]}.data 10000
- #./hos_write_complete mybucket ${test_size[$num]}.data 10000
- echo ./hos_upload_complete mybucket ${test_size[$num]}.data 10000
- ./hos_upload_complete mybucket ${test_size[$num]}.data 10000
- let "num++"
-done
-
diff --git a/example/CMakeLists.txt b/example/demo/CMakeLists.txt
index ac09873b..ac09873b 100644
--- a/example/CMakeLists.txt
+++ b/example/demo/CMakeLists.txt
diff --git a/example/data/100k.data b/example/demo/data/100k.data
index da1dfb90..da1dfb90 100644
--- a/example/data/100k.data
+++ b/example/demo/data/100k.data
Binary files differ
diff --git a/example/demo/data/10M.data b/example/demo/data/10M.data
new file mode 100644
index 00000000..6c5d4031
--- /dev/null
+++ b/example/demo/data/10M.data
Binary files differ
diff --git a/example/data/10k.data b/example/demo/data/10k.data
index 9df64990..9df64990 100644
--- a/example/data/10k.data
+++ b/example/demo/data/10k.data
Binary files differ
diff --git a/example/demo/data/11k.data b/example/demo/data/11k.data
new file mode 100644
index 00000000..258e2e6c
--- /dev/null
+++ b/example/demo/data/11k.data
Binary files differ
diff --git a/example/demo/data/12k.data b/example/demo/data/12k.data
new file mode 100644
index 00000000..909e3e3e
--- /dev/null
+++ b/example/demo/data/12k.data
Binary files differ
diff --git a/example/demo/data/13k.data b/example/demo/data/13k.data
new file mode 100644
index 00000000..f2a437ac
--- /dev/null
+++ b/example/demo/data/13k.data
Binary files differ
diff --git a/example/demo/data/14k.data b/example/demo/data/14k.data
new file mode 100644
index 00000000..e1121ab6
--- /dev/null
+++ b/example/demo/data/14k.data
Binary files differ
diff --git a/example/demo/data/15k.data b/example/demo/data/15k.data
new file mode 100644
index 00000000..c34bcd59
--- /dev/null
+++ b/example/demo/data/15k.data
Binary files differ
diff --git a/example/data/1M.data b/example/demo/data/1M.data
index 9e0f96a2..9e0f96a2 100644
--- a/example/data/1M.data
+++ b/example/demo/data/1M.data
Binary files differ
diff --git a/example/data/1k.data b/example/demo/data/1k.data
index 06d74050..06d74050 100644
--- a/example/data/1k.data
+++ b/example/demo/data/1k.data
Binary files differ
diff --git a/example/demo/data/20M.data b/example/demo/data/20M.data
new file mode 100644
index 00000000..6c5effe7
--- /dev/null
+++ b/example/demo/data/20M.data
Binary files differ
diff --git a/example/data/2M.data b/example/demo/data/2M.data
index 3301331b..3301331b 100644
--- a/example/data/2M.data
+++ b/example/demo/data/2M.data
Binary files differ
diff --git a/example/demo/data/2k.data b/example/demo/data/2k.data
new file mode 100644
index 00000000..e9784eb4
--- /dev/null
+++ b/example/demo/data/2k.data
Binary files differ
diff --git a/example/demo/data/30M.data b/example/demo/data/30M.data
new file mode 100644
index 00000000..c6fcc12d
--- /dev/null
+++ b/example/demo/data/30M.data
Binary files differ
diff --git a/example/data/3M.data b/example/demo/data/3M.data
index b7f1f882..b7f1f882 100644
--- a/example/data/3M.data
+++ b/example/demo/data/3M.data
Binary files differ
diff --git a/example/demo/data/3k.data b/example/demo/data/3k.data
new file mode 100644
index 00000000..decc0c3f
--- /dev/null
+++ b/example/demo/data/3k.data
Binary files differ
diff --git a/example/data/4M.data b/example/demo/data/4M.data
index 98fc2c0b..98fc2c0b 100644
--- a/example/data/4M.data
+++ b/example/demo/data/4M.data
Binary files differ
diff --git a/example/demo/data/4k.data b/example/demo/data/4k.data
new file mode 100644
index 00000000..08e7df17
--- /dev/null
+++ b/example/demo/data/4k.data
Binary files differ
diff --git a/example/demo/data/5M.data b/example/demo/data/5M.data
new file mode 100644
index 00000000..39953167
--- /dev/null
+++ b/example/demo/data/5M.data
Binary files differ
diff --git a/example/demo/data/5k.data b/example/demo/data/5k.data
new file mode 100644
index 00000000..e7f3c2d4
--- /dev/null
+++ b/example/demo/data/5k.data
Binary files differ
diff --git a/example/demo/data/6k.data b/example/demo/data/6k.data
new file mode 100644
index 00000000..4fdf7031
--- /dev/null
+++ b/example/demo/data/6k.data
Binary files differ
diff --git a/example/demo/data/7k.data b/example/demo/data/7k.data
new file mode 100644
index 00000000..c6d6974c
--- /dev/null
+++ b/example/demo/data/7k.data
Binary files differ
diff --git a/example/demo/data/8k.data b/example/demo/data/8k.data
new file mode 100644
index 00000000..6d17cf9d
--- /dev/null
+++ b/example/demo/data/8k.data
Binary files differ
diff --git a/example/demo/data/9k.data b/example/demo/data/9k.data
new file mode 100644
index 00000000..ba90eb00
--- /dev/null
+++ b/example/demo/data/9k.data
Binary files differ
diff --git a/example/demo/data/append.txt b/example/demo/data/append.txt
new file mode 100644
index 00000000..240e420b
--- /dev/null
+++ b/example/demo/data/append.txt
@@ -0,0 +1 @@
+this is a append test\!
diff --git a/example/demo/data/hello.txt b/example/demo/data/hello.txt
new file mode 100644
index 00000000..f2aa86d2
--- /dev/null
+++ b/example/demo/data/hello.txt
@@ -0,0 +1 @@
+hello
diff --git a/example/demo/data/hos_upload_complete b/example/demo/data/hos_upload_complete
new file mode 100755
index 00000000..045f53b2
--- /dev/null
+++ b/example/demo/data/hos_upload_complete
Binary files differ
diff --git a/example/demo/data/hos_write_complete b/example/demo/data/hos_write_complete
new file mode 100755
index 00000000..2738dc46
--- /dev/null
+++ b/example/demo/data/hos_write_complete
Binary files differ
diff --git a/example/demo/data/test_size.sh b/example/demo/data/test_size.sh
new file mode 100755
index 00000000..8051f2af
--- /dev/null
+++ b/example/demo/data/test_size.sh
@@ -0,0 +1,20 @@
+#########################################################################
+# File Name: test_times.sh
+# Author: pxz
+# Created Time: Mon 21 Sep 2020 04:43:35 PM CST
+#########################################################################
+#!/bin/bash
+test_size=("1k" "10k" "100k" "1M" "2M" "3M" "4M" "5M" "10M" "20M" "30M")
+#test_size=("1k" "2k" "3k" "4k" "5k" "6k" "7k" "8k" "9k" "10k" "11k" "12k" "13k" "14k" "15k")
+num=0
+
+#while((${num} < 15))
+while((${num} < 11))
+do
+ echo ./hos_write_complete mybucket ${test_size[$num]}.data 1000
+ ./hos_write_complete mybucket ${test_size[$num]}.data 1000
+ #echo ./hos_upload_complete mybucket ${test_size[$num]}.data 1000
+ #./hos_upload_complete mybucket ${test_size[$num]}.data 1000
+ let "num++"
+done
+
diff --git a/example/demo/data/test_times.sh b/example/demo/data/test_times.sh
new file mode 100755
index 00000000..a9bab6fc
--- /dev/null
+++ b/example/demo/data/test_times.sh
@@ -0,0 +1,16 @@
+#########################################################################
+# File Name: test_times.sh
+# Author: pxz
+# Created Time: Mon 21 Sep 2020 04:43:35 PM CST
+#########################################################################
+#!/bin/bash
+test_times=(1, 10, 100, 1000, 10000)
+num=0
+
+while((${num} < 5))
+do
+ echo ./hos_write_complete mybucket 1k.data ${test_times[$num]}
+ ./hos_write_complete mybucket 1k.data ${test_times[$num]}
+ let "num++"
+done
+
diff --git a/example/hos_upload_complete.cpp b/example/demo/hos_upload_complete.cpp
index 84289b13..a32465a8 100644
--- a/example/hos_upload_complete.cpp
+++ b/example/demo/hos_upload_complete.cpp
@@ -11,7 +11,7 @@ extern "C"
#include<string.h>
#include<time.h>
}
-#include"../src/hos_client.h"
+#include"hos_client.h"
//#define test_times 10000
@@ -82,7 +82,7 @@ int main(int argc, char *argv[])
file_to_buffer(object, buf, &buf_size);
debuginfo("hos_client_init start ...\n");
- hos_client_handle handle = hos_client_create("http://192.168.44.10:9098/hos/", "default", "default", 4);
+ hos_client_handle handle = hos_client_create("http://192.168.44.10:9098/hos/", "default", "default", 4, 3000);
if (handle == NULL)
{
debuginfo("error:hos_client_handle\n");
diff --git a/example/hos_write_complete.cpp b/example/demo/hos_write_complete.cpp
index ebe13fbc..615fd81f 100644
--- a/example/hos_write_complete.cpp
+++ b/example/demo/hos_write_complete.cpp
@@ -11,7 +11,7 @@ extern "C"
#include<string.h>
#include<time.h>
}
-#include"../src/hos_client.h"
+#include"hos_client.h"
//#define test_times 10000
@@ -71,21 +71,23 @@ int main(int argc, char *argv[])
char *object = argv[2];
int test_times = atoi(argv[3]);
//int test_times = 10000;
- //char *buf = (char *)malloc(1024 * 1024 * 4);
- char buf[1024 * 1024 * 4];
+ char *buf = (char *)malloc(1024 * 1024 * 40);
+ //char buf[1024 * 1024 * 4];
//char buf[1024 * 4];
size_t buf_size;
int mode = FILE_MODE;
- size_t fd[10000] = {0};
+ size_t fd[10001] = {0};
userdata_t data = {&finished};
+ //printf("finished:%lu\n", finished.tv_sec * 1000 * 1000 * 1000 + finished.tv_nsec);
file_to_buffer(object, buf, &buf_size);
debuginfo("hos_client_init start ...\n");
- hos_client_handle handle = hos_client_create("http://192.168.44.10:9098/hos/", "default", "default", 4);
+ hos_client_handle handle = hos_client_create("http://192.168.40.223:9098/hos/", "default", "default", 4, 4000);
+ //hos_client_handle handle = hos_client_create("http://192.168.32.10:9098/hos/", "default", "default", 4);
if (handle == NULL)
{
- debuginfo("error:hos_client_handle\n");
+ printf("error:hos_client_handle\n");
return -1;
}
debuginfo("hos_client_init success ... \n");
@@ -93,7 +95,7 @@ int main(int argc, char *argv[])
debuginfo("hos_create_bucket start ... \n");
if(hos_create_bucket(handle, bucket))
{
- debuginfo("hos_create_bucket failed ... \n");
+ printf("hos_create_bucket failed ... \n");
return -1;
}
debuginfo("hos_create_bucket success ... \n");
@@ -101,22 +103,23 @@ int main(int argc, char *argv[])
debuginfo("hos_verify_bucket start ... \n");
if(!hos_verify_bucket(handle, bucket))
{
- debuginfo("hos_verify_bucket failed ... \n");
+ printf("hos_verify_bucket failed ... \n");
return -1;
}
debuginfo("hos_verify_bucket success ... \n");
#if 1
mode = FILE_MODE;
+#if 0
for (i = 0; i < test_times; i++)
{
- fd[i] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode);
}
-
+#endif
debuginfo("hos_upload_file start ...\n");
clock_gettime(CLOCK_MONOTONIC, &start);
for (i = 0; i < test_times; i++)
{
+ fd[i] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode);
hos_write(fd[i], object, 0, 0, 0);
}
clock_gettime(CLOCK_MONOTONIC, &end);
@@ -127,15 +130,22 @@ int main(int argc, char *argv[])
#else
mode = BUFF_MODE | APPEND_MODE;
+#if 0
for (i = 0; i < test_times; i++)
{
- fd[i] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode);
}
+#endif
debuginfo("hos_upload_buf start ...\n");
clock_gettime(CLOCK_MONOTONIC, &start);
for (i = 0; i < test_times; i++)
{
- hos_write(fd[i], buf, buf_size, 0, i);
+ fd[i] = hos_open_fd(handle, bucket, object, callback, (void *)&data, 0, mode);
+ //printf("hos_write start...\n");
+ if(hos_write(fd[i], buf, buf_size, 0, i + 1) != HOS_CLIENT_OK)
+ {
+ printf("error: hos_write!\n");
+ }
+ //printf("hos_write success...\n");
}
clock_gettime(CLOCK_MONOTONIC, &end);
time = calc_time(start, end);
@@ -147,12 +157,14 @@ int main(int argc, char *argv[])
debuginfo("hos_client_close start ...\n");
if (hos_client_destory(handle) == 0)
{
+ //printf("start:%lu\n", start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec);
+ //printf("finished:%lu\n", finished.tv_sec * 1000 * 1000 * 1000 + finished.tv_nsec);
time = calc_time(start, finished);
time /= test_times;
printf("hos upload finished spent %llu ns\n", time);
}
debuginfo("hos_client_close end ...\n");
-
+ free(buf);
return 0;
}
diff --git a/example/test_size.sh b/example/demo/test_size.sh
index 3884091e..3884091e 100755
--- a/example/test_size.sh
+++ b/example/demo/test_size.sh
diff --git a/example/test_times.sh b/example/demo/test_times.sh
index 6e1adbc4..6e1adbc4 100755
--- a/example/test_times.sh
+++ b/example/demo/test_times.sh
diff --git a/example/performance/CMakeLists.txt b/example/performance/CMakeLists.txt
new file mode 100644
index 00000000..1eea1e3a
--- /dev/null
+++ b/example/performance/CMakeLists.txt
@@ -0,0 +1,13 @@
+cmake_minimum_required(VERSION 3.5)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
+set(CMAKE_BUILD_TYPE Debug)
+project(HosClientPerformance)
+
+link_directories(/usr/local/lib64/)
+link_directories(/opt/MESA/lib/)
+link_libraries(hos-client-cpp pthread)
+include_directories(/opt/MESA/include)
+
+add_executable(HosClientPerformance HosClientPerformance.cpp)
+target_link_libraries(HosClientPerformance hos-client-cpp)
+
diff --git a/example/performance/HosClientPerformance.cpp b/example/performance/HosClientPerformance.cpp
new file mode 100644
index 00000000..a445cc75
--- /dev/null
+++ b/example/performance/HosClientPerformance.cpp
@@ -0,0 +1,542 @@
+/*************************************************************************
+ > File Name: HosClientPerformance.cpp
+ > Author: pxz
+ > Created Time: Sat 10 Oct 2020 05:26:02 PM CST
+ ************************************************************************/
+extern "C"
+{
+#include<stdio.h>
+#include<stdlib.h>
+#include<unistd.h>
+#include<string.h>
+#include<time.h>
+#include<pthread.h>
+#include<dirent.h>
+#include<sys/stat.h>
+}
+#include"../../src/hos_client.h"
+
+#define MAX_THREAD_NUM 16
+#ifndef MIN
+#define MIN(a,b) ((a) > (b)) ? (b) : (a)
+#endif
+
+typedef struct conf_s
+{
+#define STRING_SIZE 128
+ char endpoint[STRING_SIZE];
+ char bucket[STRING_SIZE];
+ char object[STRING_SIZE];
+ char file[STRING_SIZE];
+ size_t pool_size;
+ size_t thread_sum;
+ size_t append_size;
+ int mode;
+}conf_t;
+
+typedef struct thread_info_s
+{
+ conf_t conf;
+ hos_client_handle handle;
+ size_t thread_num;
+}thread_info_t;
+
+static void configuration_init(conf_t *conf)
+{
+ strcpy(conf->endpoint, "http://192.168.40.223:9098/hos/");
+ strcpy(conf->bucket, "mybucket");
+ strcpy(conf->object, "myobject");
+ strcpy(conf->file, "./file/test.txt");
+ conf->pool_size = 4000;
+ conf->append_size = 1024;
+ conf->thread_sum = 1;
+ conf->mode = BUFF_MODE;
+
+ return ;
+}
+
+typedef struct userdata_s
+{
+ struct timespec *finished;
+}userdata_t;
+
+static size_t calc_time(struct timespec start, struct timespec end)
+{
+ return (end.tv_sec * 1000 * 1000 * 1000 + end.tv_nsec -
+ (start.tv_sec * 1000 * 1000 * 1000 + start.tv_nsec));
+}
+
+int read_file_list(const char *path, char file_name[][256])
+{
+ DIR *dir;
+ struct dirent *ptr;
+ int path_len = strlen(path);
+ int file_num = 0;
+
+ if ((dir=opendir(path)) == NULL)
+ {
+ perror("Open dir error...");
+ exit(-1);
+ }
+
+ while (((ptr=readdir(dir)) != NULL) && (file_num < 100))
+ {
+ if(strcmp(ptr->d_name,".")==0 || strcmp(ptr->d_name,"..")==0) ///current dir OR parrent dir
+ continue;
+ else if((ptr->d_type == 8) || (ptr->d_type == 10))
+ {
+ memcpy(file_name[file_num], path, path_len);
+ strcat(file_name[file_num], ptr->d_name);
+ }
+ else if(ptr->d_type == 4) ///dir
+ {
+ continue;
+ }
+ file_num++;
+ }
+ closedir(dir);
+ return 0;
+}
+
+static void callback(bool result, const char *error, void *userdata)
+{
+#if 0
+ userdata_t *data = (userdata_t *)userdata;
+ clock_gettime(CLOCK_MONOTONIC, data->finished);
+#endif
+ return ;
+}
+
+static int file_to_buffer(const char *file, char *buffer, size_t *len)
+{
+ FILE *fp = fopen(file, "r");
+ int num = 0;
+ *len = 0;
+ if (fp == NULL)
+ {
+ printf("fopen file failed:%s\n", file);
+ return -1;
+ }
+ do{
+ num = fread(&buffer[*len], 1, 4096, fp);
+ if (num < 0)
+ {
+ return -1;
+ }
+ *len += num;
+ }while(num == 4096);
+ fclose(fp);
+ return 0;
+}
+
+static int upload_file(char *file, char *buff, int buff_len, int test_times, thread_info_t *thread_info, char *performance_info)
+{
+ int i;
+ FILE *fp = NULL;
+ size_t fd[3000];
+ conf_t *conf = &thread_info->conf;
+ struct timespec tstart, tend, twrite;
+ size_t time_write, time_upload;
+ size_t len = strlen(performance_info);
+ char file_size[128];
+ //写文件
+
+ clock_gettime(CLOCK_MONOTONIC, &tstart);
+ for (i = 0; i < test_times; i++)
+ {
+ fp = fopen(file, "w+");
+ if (fp == NULL)
+ {
+ printf("error:fopen failed\n");
+ return -1;
+ }
+ if (fwrite(buff, buff_len, 1, fp) != 1)
+ {
+ printf("error:fwrite failed\n");
+ fclose(fp);
+ return -1;
+ }
+ fclose(fp);
+ }
+ clock_gettime(CLOCK_MONOTONIC, &twrite);
+ time_write = calc_time(tstart, twrite);
+ time_write /= test_times;
+
+ //上传文件
+ clock_gettime(CLOCK_MONOTONIC, &tstart);
+ for (i = 0; i < test_times; i++)
+ {
+ fd[i] = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode);
+ if (hos_write(fd[i], file, 0, thread_info->thread_num, 0) != HOS_CLIENT_OK)
+ {
+ printf("error:hos_write file:%s\n", file);
+ return -1;
+ }
+ }
+ clock_gettime(CLOCK_MONOTONIC, &tend);
+ time_upload = calc_time(tstart, tend);
+ time_upload /= test_times;
+
+ sprintf(file_size, "%dk", buff_len / 1024);
+ sprintf(&performance_info[len], "%-20lu%-20s%-20lu%-20lu%-20lu\n",
+ thread_info->thread_num, file_size, time_write, time_upload, time_write + time_upload);
+
+ //hos_close_fd(fd, thread_info->thread_num);
+
+ return 0;
+}
+
+static int upload_buff(char * buff, int buff_len, int test_times, thread_info_t *thread_info, char *performance_info)
+{
+ int i = 0;
+ int j = 0;
+ size_t fd = 0;
+ size_t tmp = 0;
+ size_t rest = 0;
+ struct timespec tstart, ttmp;
+ size_t time = 0;
+ size_t len;
+ conf_t *conf = &thread_info->conf;
+ char file_size[128];
+ char append_size[128];
+
+ fd = hos_open_fd(thread_info->handle, conf->bucket, conf->object, callback, NULL, thread_info->thread_num, conf->mode);
+#if 0
+ size_t time_tmp = 0;
+ while(1)
+ {
+ tmp = j * conf->append_size;
+ rest = buff_len - tmp;
+ if (rest < conf->append_size)
+ {
+ clock_gettime(CLOCK_MONOTONIC, &tstart);
+ for (i = 0; i < test_times; i++)
+ {
+ hos_write(fd, &buff[tmp], rest, thread_info->thread_num, j + 1);
+ }
+ clock_gettime(CLOCK_MONOTONIC, &ttmp);
+ time_tmp = calc_time(tstart, ttmp);
+ time_tmp /= test_times;
+ time += time_tmp;
+
+ sprintf(file_size, "%dk", buff_len / 1024);
+ sprintf(append_size, "%luk", conf->append_size / 1024);
+ len = strlen(performance_info);
+ sprintf(&performance_info[len], "%-20lu%-20s%-20s%-20lu%-20lu\n",
+ thread_info->thread_num, file_size, append_size, time_tmp, time);
+ break;
+ }
+ clock_gettime(CLOCK_MONOTONIC, &tstart);
+ for (i = 0; i < test_times; i++)
+ {
+ hos_write(fd, &buff[tmp], conf->append_size, thread_info->thread_num, j + 1);
+ }
+ clock_gettime(CLOCK_MONOTONIC, &ttmp);
+ time_tmp = calc_time(tstart, ttmp);
+ time_tmp /= test_times;
+ time += time_tmp;
+
+ sprintf(file_size, "%dk", buff_len / 1024);
+ sprintf(append_size, "%luk", conf->append_size / 1024);
+ len = strlen(performance_info);
+ sprintf(&performance_info[len], "%-20lu%-20s%-20s%-20lu%-20lu\n",
+ thread_info->thread_num, file_size, append_size, time_tmp, time);
+
+ j++;
+ }
+#else
+ if (conf->mode & APPEND_MODE)
+ {
+ clock_gettime(CLOCK_MONOTONIC, &tstart);
+ for (i = 0; i < test_times; i++)
+ {
+ while(1)
+ {
+ tmp = j * conf->append_size;
+ rest = buff_len - tmp;
+ if (rest < conf->append_size)
+ {
+ hos_write(fd, &buff[tmp], rest, thread_info->thread_num, j + 1);
+ break;
+ }
+ hos_write(fd, &buff[tmp], conf->append_size, thread_info->thread_num, j + 1);
+ j++;
+ }
+ }
+ clock_gettime(CLOCK_MONOTONIC, &ttmp);
+ time = calc_time(tstart, ttmp);
+ time /= test_times;
+
+ sprintf(file_size, "%dk", buff_len / 1024);
+ sprintf(append_size, "%luk", conf->append_size / 1024);
+ len = strlen(performance_info);
+ sprintf(&performance_info[len], "%-20lu%-20s%-20s%-20d%-20lu\n",
+ thread_info->thread_num, file_size, append_size, 0, time);
+ }else
+ {
+ clock_gettime(CLOCK_MONOTONIC, &tstart);
+ for (i = 0; i < test_times; i++)
+ {
+ hos_write(fd, buff, buff_len, thread_info->thread_num, j + 1);
+ }
+ clock_gettime(CLOCK_MONOTONIC, &ttmp);
+ time = calc_time(tstart, ttmp);
+ time /= test_times;
+
+ sprintf(file_size, "%dk", buff_len / 1024);
+ sprintf(append_size, "%luk", conf->append_size / 1024);
+ len = strlen(performance_info);
+ sprintf(&performance_info[len], "%-20lu%-20s%-20d%-20d%-20lu\n",
+ thread_info->thread_num, file_size, 0, 0, time);
+ }
+
+
+#endif
+ hos_close_fd(fd, thread_info->thread_num);
+
+ return 0;
+}
+
+static void *put_object_thread(void *ptr)
+{
+ char *performance_info = NULL;
+ thread_info_t *thread_info = (thread_info_t *)ptr;
+ conf_t *conf = &thread_info->conf;
+ char file[128];
+ size_t buff_len;
+ int ret;
+ int i;
+ char *buff = NULL;
+ char file_name[100][256];
+ struct stat s_buf;
+ int test_times = 1;
+
+ buff = (char *)malloc(30 * 1024 * 1024);
+ if (buff == NULL)
+ {
+ perror(" ");
+ pthread_exit(NULL);
+ }
+
+ performance_info = (char *)malloc(1024 * 1024);
+ if (performance_info == NULL)
+ {
+ perror(" ");
+ free(buff);
+ pthread_exit(NULL);
+ }
+ memset(performance_info, 0, 10240);
+ memset(file_name, 0, 100 *256);
+
+ stat(conf->file, &s_buf);
+ if (S_ISDIR(s_buf.st_mode))
+ {
+ read_file_list(conf->file, file_name);
+ for (i = 0; i < 100; i++)
+ {
+ if (file_name[i][0] == '\0')
+ break;
+ ret = file_to_buffer(file_name[i], buff, &buff_len);
+ if (ret == -1)
+ {
+ free(buff);
+ free(performance_info);
+ pthread_exit(NULL);
+ }
+ if (conf->mode & BUFF_MODE)
+ {
+ upload_buff(buff, buff_len, test_times, thread_info, performance_info);
+ }else
+ {
+ sprintf(file, "./file/file_%lu_%d", thread_info->thread_num, i);
+ upload_file(file, buff, buff_len, test_times, thread_info, performance_info);
+ }
+ }
+ }else
+ {
+ ret = file_to_buffer(conf->file, buff, &buff_len);
+ if (ret == -1)
+ {
+ free(buff);
+ free(performance_info);
+ pthread_exit(NULL);
+ }
+
+ if (conf->mode & BUFF_MODE)
+ {
+ upload_buff(buff, buff_len, test_times, thread_info, performance_info);
+ }else
+ {
+ sprintf(file, "./file/file_%lu", thread_info->thread_num);
+ upload_file(file, buff, buff_len, test_times, thread_info, performance_info);
+ }
+ }
+ free(buff);
+ pthread_exit(performance_info);
+}
+
+int main(int argc, char *argv[])
+{
+ int ch;
+ int buf_size;
+ conf_t conf;
+ char *object;
+ char *retval;
+ size_t thread_num;
+ size_t thread[MAX_THREAD_NUM];
+ thread_info_t thread_info[MAX_THREAD_NUM];
+ cpu_set_t mask;
+ FILE *log = NULL;
+ char log_name[256];
+ const char *log_prefix = "./log/";
+ time_t timep;
+#if 0
+ if (argc <= 1)
+ {
+ printf("usage: HosClientPerformance \n[-e set endpoint] \n[-b set bucket] \n"
+ "[-o set object] \n[-f set file] \n[-p set pool size] \n"
+ "[-t set thread sum] \n[-B set BUFF_MODE] \n"
+ "[-F set FILE_MODE] \n[-A set APPEND_MODE] \n"
+ "[-h show help info] \n");
+ return -1;
+ }
+#endif
+ configuration_init(&conf);
+ //读取命令行配置
+ while((ch = getopt(argc, argv, "a:e:b:o:f:p:t:k:BFAh")) != -1)
+ {
+ switch(ch)
+ {
+ case 'a':
+ conf.append_size = 1024 * atoi(optarg);
+ break;
+ case 'e':
+ //endpoint
+ buf_size = MIN(STRING_SIZE, strlen(optarg));
+ strncpy((char *)conf.endpoint, optarg, buf_size);
+ break;
+ case 'b':
+ buf_size = MIN(STRING_SIZE, strlen(optarg));
+ strncpy((char *)conf.bucket, optarg, buf_size);
+ conf.bucket[buf_size] = '\0';
+ break;
+ case 'o':
+ buf_size = MIN(STRING_SIZE, strlen(optarg));
+ strncpy(conf.object, optarg, buf_size);
+ conf.object[buf_size] = '\0';
+ break;
+ case 'f':
+ buf_size = MIN(STRING_SIZE, strlen(optarg));
+ strncpy(conf.file, optarg, buf_size);
+ conf.file[buf_size] = '\0';
+ break;
+ case 'p':
+ conf.pool_size = atoi(optarg);
+ conf.pool_size = MIN(3000, conf.pool_size);
+ break;
+ case 't':
+ conf.thread_sum = atoi(optarg);
+ break;
+ case 'B':
+ conf.mode &= BUFF_MODE;
+ break;
+ case 'F':
+ conf.mode &= FILE_MODE;
+ break;
+ case 'A':
+ conf.mode |= APPEND_MODE;
+ break;
+ case 'k':
+ conf.append_size = 1024 * atoi(optarg);
+ break;
+ case 'h':
+ default:
+ printf("usage: HosClientPerformance \n[-e set endpoint] \n[-b set bucket] \n"
+ "[-o set object] \n[-f set file] \n[-p set pool size] \n"
+ "[-t set thread sum] \n[-B set BUFF_MODE] \n"
+ "[-F set FILE_MODE] \n[-A set APPEND_MODE] \n"
+ "[-h show help info] \n[-a set append size(K)]\n");
+ return -1;
+ break;
+ }
+ }
+
+ strcpy(log_name, log_prefix);
+ time(&timep);
+ strftime(&log_name[strlen(log_prefix)], sizeof(log_name) - strlen(log_prefix),"%Y%m%d%H%M%S.log", localtime(&timep));
+ log = fopen(log_name, "a+");
+ if (log == NULL)
+ {
+ perror(log_name);
+ return -1;
+ }
+ //创建client
+ hos_client_handle handle = hos_client_create(conf.endpoint, "default", "default", conf.thread_sum, conf.pool_size);
+ if (handle == NULL)
+ {
+ printf("error:hos_client_handle\n");
+ fclose(log);
+ return -1;
+ }
+
+ //创建bucket
+ if (hos_create_bucket(handle, conf.bucket))
+ {
+ printf("error:hos_create_bucket\n");
+ fclose(log);
+ hos_client_destory(handle);
+ return -1;
+ }
+
+ printf("\n==============================================================================================================================\n");
+ if (conf.mode & BUFF_MODE)
+ {
+ printf("%-20s%-20s%-20s%-20s%-20s\n", "thread_id", "file_size", "append_size", "upload_time", "total_time");
+ }else
+ {
+ printf("%-20s%-20s%-20s%-20s%-20s\n", "thread_id", "file_size", "write_time", "upload_time", "total_time");
+ }
+ for ( thread_num = 0; thread_num < conf.thread_sum; thread_num++ )
+ {
+ thread_info[thread_num].conf = conf;
+ object = thread_info[thread_num].conf.object;
+ sprintf(&object[strlen(object)], "-%lu", thread_num);
+ thread_info[thread_num].thread_num = thread_num;
+ thread_info[thread_num].handle = handle;
+ if(pthread_create(&thread[thread_num], NULL, put_object_thread, (void *)&thread_info[thread_num]))
+ {
+ perror(" ");
+ fclose(log);
+ hos_client_destory(handle);
+ return -1;
+ }
+
+ CPU_ZERO(&mask);
+ CPU_SET(thread_num + 8, &mask);
+ if (pthread_setaffinity_np(thread[thread_num], sizeof(mask), &mask) != 0)
+ {
+ printf("warning:could not set CPU affinity, continuing...\n");
+ }
+ }
+
+ for (thread_num = 0; thread_num < conf.thread_sum; thread_num++)
+ {
+ pthread_join(thread[thread_num], (void **)&retval);
+ if (retval)
+ {
+ printf("%s", retval);
+ fwrite(retval, strlen(retval), 1, log);
+ free(retval);
+ }
+ }
+
+ if (hos_client_destory(handle) == 0)
+ {
+ //time = calc_time(start, finished);
+ //time /= test_times;
+ //printf("hos upload finished spent %llu ns\n", time);
+ }
+
+ fclose(log);
+ return 0;
+}
diff --git a/example/performance/file/100k.data b/example/performance/file/100k.data
new file mode 100644
index 00000000..da1dfb90
--- /dev/null
+++ b/example/performance/file/100k.data
Binary files differ
diff --git a/example/performance/file/10M.data b/example/performance/file/10M.data
new file mode 100644
index 00000000..6c5d4031
--- /dev/null
+++ b/example/performance/file/10M.data
Binary files differ
diff --git a/example/performance/file/10M.txt b/example/performance/file/10M.txt
new file mode 100644
index 00000000..6c5d4031
--- /dev/null
+++ b/example/performance/file/10M.txt
Binary files differ
diff --git a/example/performance/file/10k.data b/example/performance/file/10k.data
new file mode 100644
index 00000000..9df64990
--- /dev/null
+++ b/example/performance/file/10k.data
Binary files differ
diff --git a/example/performance/file/1M.data b/example/performance/file/1M.data
new file mode 100644
index 00000000..9e0f96a2
--- /dev/null
+++ b/example/performance/file/1M.data
Binary files differ
diff --git a/example/performance/file/1k.data b/example/performance/file/1k.data
new file mode 100644
index 00000000..06d74050
--- /dev/null
+++ b/example/performance/file/1k.data
Binary files differ
diff --git a/example/performance/file/20M.data b/example/performance/file/20M.data
new file mode 100644
index 00000000..6c5effe7
--- /dev/null
+++ b/example/performance/file/20M.data
Binary files differ
diff --git a/example/performance/file/2M.data b/example/performance/file/2M.data
new file mode 100644
index 00000000..3301331b
--- /dev/null
+++ b/example/performance/file/2M.data
Binary files differ
diff --git a/example/performance/file/30M.data b/example/performance/file/30M.data
new file mode 100644
index 00000000..c6fcc12d
--- /dev/null
+++ b/example/performance/file/30M.data
Binary files differ
diff --git a/example/performance/file/3M.data b/example/performance/file/3M.data
new file mode 100644
index 00000000..b7f1f882
--- /dev/null
+++ b/example/performance/file/3M.data
Binary files differ
diff --git a/example/performance/file/4M.data b/example/performance/file/4M.data
new file mode 100644
index 00000000..98fc2c0b
--- /dev/null
+++ b/example/performance/file/4M.data
Binary files differ
diff --git a/example/performance/file/5M.data b/example/performance/file/5M.data
new file mode 100644
index 00000000..39953167
--- /dev/null
+++ b/example/performance/file/5M.data
Binary files differ
diff --git a/example/performance/file/c.sh b/example/performance/file/c.sh
new file mode 100755
index 00000000..974d0741
--- /dev/null
+++ b/example/performance/file/c.sh
@@ -0,0 +1,15 @@
+#########################################################################
+# File Name: c.sh
+# Author: pxz
+# Created Time: Tue 13 Oct 2020 06:28:57 PM CST
+#########################################################################
+#!/bin/bash
+size=("1k" "10k" "100k" "1M" "2M" "3M" "4M" "5M" "10M" "20M" "30M")
+
+num=0
+
+while((${num} < 11))
+do
+ truncate -s ${size[$num]} ${size[$num]}.data
+ let "num++"
+done
diff --git a/example/performance/file/file/1024k.data b/example/performance/file/file/1024k.data
new file mode 100644
index 00000000..9e0f96a2
--- /dev/null
+++ b/example/performance/file/file/1024k.data
Binary files differ
diff --git a/example/performance/file/file/1M.data b/example/performance/file/file/1M.data
new file mode 100644
index 00000000..9e0f96a2
--- /dev/null
+++ b/example/performance/file/file/1M.data
Binary files differ
diff --git a/example/performance/file/file_0 b/example/performance/file/file_0
new file mode 100644
index 00000000..06d74050
--- /dev/null
+++ b/example/performance/file/file_0
Binary files differ
diff --git a/example/performance/file/test.txt b/example/performance/file/test.txt
new file mode 100644
index 00000000..93f84a69
--- /dev/null
+++ b/example/performance/file/test.txt
Binary files differ
diff --git a/example/performance/test.sh b/example/performance/test.sh
new file mode 100755
index 00000000..1c18bc39
--- /dev/null
+++ b/example/performance/test.sh
@@ -0,0 +1,17 @@
+#########################################################################
+# File Name: test.sh
+# Author: pxz
+# Created Time: Tue 13 Oct 2020 02:28:31 PM CST
+#########################################################################
+#!/bin/bash
+size=(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 200, 300, 400, 500)
+
+num=0
+
+while((${num} < 37))
+do
+ #./HosClientPerformance -a ${size[$num]}
+ echo "./HosClientPerformance -a ${size[$num]} -A -f ./file/1M.data"
+ ./HosClientPerformance -a ${size[$num]} -A -f ./file/1M.data
+ let "num++"
+done
diff --git a/src/hos_client.cpp b/src/hos_client.cpp
index f49bf9ff..b4e91867 100644
--- a/src/hos_client.cpp
+++ b/src/hos_client.cpp
@@ -80,14 +80,14 @@ static void PutObjectAsyncFinished(const Aws::S3::S3Client* S3Client,
}
}
-hos_client_handle hos_client_create(const char *endpoint, const char *accesskeyid, const char *secretkey, size_t thread_sum)
+hos_client_handle hos_client_create(const char *endpoint, const char *accesskeyid, const char *secretkey, size_t thread_sum, size_t pool_size)
{
if (!endpoint || !accesskeyid || !secretkey || thread_sum > MAX_THREAD_NUM)
{
return NULL;
}
Aws::SDKOptions options;
- //auto options = new Aws::SDKOptions;
+ //options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
Aws::InitAPI(options);
hos_client_handle handle = (hos_client_handle)malloc(sizeof(hos_client_handle_t));
@@ -99,7 +99,7 @@ hos_client_handle hos_client_create(const char *endpoint, const char *accesskeyi
config.endpointOverride = endpoint;
config.verifySSL = false;
config.enableEndpointDiscovery = true;
- config.executor = std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>(std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(100));//支持线程池
+ config.executor = std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>(std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(pool_size));//支持线程池
handle->S3Client = new Aws::S3::S3Client(credentials, config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, false);
handle->append_size = 30 * 1024 * 1024;
@@ -192,7 +192,7 @@ static int hos_upload_stream(hos_client_handle handle, const char *bucket, const
//设置上传数据类型
if (file_type == 0)
{
- if (stat(object, &buffer) == -1)
+ if (stat(data, &buffer) == -1)
{
return HOS_FILE_NOT_EXITS;
}
@@ -317,8 +317,7 @@ int hos_write(size_t fd, const char *stream, size_t stream_len, size_t thread_id
}
else
{
- //BUFF_MODE
- if (stat(hos_info->object, &buffer) == -1)
+ if (stat(stream, &buffer) == -1)
{
return HOS_FILE_NOT_EXITS;
}
diff --git a/src/hos_client.h b/src/hos_client.h
index d1cb7db5..668af9fd 100644
--- a/src/hos_client.h
+++ b/src/hos_client.h
@@ -80,9 +80,9 @@ typedef void (*put_finished_callback)(bool, const char *, void *);
* const char *accesskeyid AWS access key ID,如”default“
* const char *secretkey AWS secret key,如”default“
* size_t thread_sum 线程总数
- * 返回值: 成功返回一个非空句柄,失败返回NULL。(失败原因都是因为输入参数为空)
+ * 返回值: 成功返回一个非空句柄,失败返回NULL。(失败原因都是因为输入参数不合法)
*************************************************************************************/
-hos_client_handle hos_client_create(const char *endpoint, const char *accesskeyid, const char *secretkey, size_t thread_sum);
+hos_client_handle hos_client_create(const char *endpoint, const char *accesskeyid, const char *secretkey, size_t thread_sum, size_t pool_size);
/*************************************************************************************
* 函数名: hos_create_bucket
* 参数: hos_client_handle handle 非空句柄