应用EAL初始化后，恢复线程的亲和性设置。v4.2.20-20170922v4.2.19-20170921

- 获得应用初始化程的亲和性设置，EAL初始化后恢复，避免从此线程派生的线程全部带有EAL设置的亲和性。

1 files changed, 130 insertions, 119 deletions

diff --git a/app/src/marsio.c b/app/src/marsio.c
index 1bcfe08..16fa589 100644
--- a/app/src/marsio.c
+++ b/app/src/marsio.c
@@ -32,15 +32,15 @@ struct mr_instance * _current_instance = NULL;
 
 /* 写入Command参数 */
 static void __write_arg(char * eal_argv[], unsigned int * eal_argc,
-	unsigned int max_argc, const char * value)
+    unsigned int max_argc, const char * value)
 {
-	assert(max_argc >= *eal_argc);
-	char * mem = (char *)malloc(MR_STRING_MAX * sizeof(char));
-	assert(mem != NULL);
+    assert(max_argc >= *eal_argc);
+    char * mem = (char *)malloc(MR_STRING_MAX * sizeof(char));
+    assert(mem != NULL);
 
-	snprintf(mem, MR_STRING_MAX * sizeof(char), "%s", value);
-	eal_argv[(*eal_argc)++] = mem;
-	return;
+    snprintf(mem, MR_STRING_MAX * sizeof(char), "%s", value);
+    eal_argv[(*eal_argc)++] = mem;
+    return;
 }
 
 #define WRITE_ARG(x)													\
@@ -50,15 +50,15 @@ static void __write_arg(char * eal_argv[], unsigned int * eal_argc,
 /* EAL环境初始化 */
 static void mrapp_eal_init(struct mr_instance * instance)
 {
-	char * eal_argv[MR_LIB_MAX_EAL_ARGC];
-	unsigned int eal_argc = 0;
+    char * eal_argv[MR_LIB_MAX_EAL_ARGC];
+    unsigned int eal_argc = 0;
 
     if (g_eal_inited > 0) return;
 
     WRITE_ARG(instance->appsym);
-	WRITE_ARG("-c");
-	WRITE_ARG("0x1");
-	WRITE_ARG("--proc-type=secondary");
+    WRITE_ARG("-c");
+    WRITE_ARG("0x1");
+    WRITE_ARG("--proc-type=secondary");
 
     char str_virtaddr[MR_STRING_MAX];
     int ret = MESA_load_profile_string_nodef(instance->g_cfgfile_path, "eal",
@@ -72,7 +72,7 @@ static void mrapp_eal_init(struct mr_instance * instance)
 
     // DPDK和SYSTEMD的日志级别差1
     unsigned int loglevel = g_logger_level + 1;
-    MESA_load_profile_uint_def(instance->g_cfgfile_path, "eal", "loglevel", 
+    MESA_load_profile_uint_def(instance->g_cfgfile_path, "eal", "loglevel",
         &loglevel, loglevel);
 
     /* 检查日志选项，必须在1~8之间 */
@@ -83,9 +83,9 @@ static void mrapp_eal_init(struct mr_instance * instance)
     }
 
 #if RTE_VERSION >= RTE_VERSION_NUM(17,5,0,0)
-	rte_log_set_global_level(loglevel);
+    rte_log_set_global_level(loglevel);
 #else
-	rte_set_log_level(loglevel);
+    rte_set_log_level(loglevel);
 #endif
 
     g_logger_level = loglevel - 1;
@@ -107,12 +107,23 @@ static void mrapp_eal_init(struct mr_instance * instance)
 
     MR_INFO("EAL Parameters: %s", str_eal_cmdline);
 
+    /* 获得当前线程的亲和性设置，EAL初始化后恢复，避免从此线程派生的
+       线程全部带有EAL设置的亲和性。 */
+
+    cpu_set_t __cpu_set;
+    ret = pthread_getaffinity_np(pthread_self(), sizeof(cpu_set_t), &__cpu_set);
+    MR_VERIFY_2(ret >= 0, "Cannot get init thread affinity: %s", strerror(errno));
+        
     rte_openlog_stream(stderr);
-	ret = rte_eal_init(eal_argc, eal_argv);
-	MR_VERIFY_2(ret >= 0, "Cannot init EAL Enviorment, Failed");
+    ret = rte_eal_init(eal_argc, eal_argv);
+    MR_VERIFY_2(ret >= 0, "Cannot init EAL Enviorment, Failed");
 
+    ret = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &__cpu_set);
+    MR_VERIFY_2(ret >= 0, "Cannot set init thread affinity: %s", strerror(errno));
+    
+    /* EAL环境初始化成功 */
     g_eal_inited = 1;
-	return;
+    return;
 }
 
 static int mrapp_ctrlmsg_init(struct mr_instance * instance)
@@ -146,79 +157,79 @@ static int mrapp_ctrlmsg_init(struct mr_instance * instance)
 
 static int mrapp_distributer_init(struct mr_instance * instance)
 {
-	unsigned int distmode = LDBC_DIST_OUTER_TUPLE2;
-	unsigned int hashmode = LDBC_HASH_SYM_CRC;
-
-	MESA_load_profile_uint_def(instance->g_cfgfile_path, "service", "distmode",
-		&distmode, LDBC_DIST_OUTER_TUPLE2);
-	MESA_load_profile_uint_def(instance->g_cfgfile_path, "service", "hashmode",
-		&hashmode, LDBC_HASH_SYM_CRC);
-
-	if (distmode < 0 || distmode >= LDBC_DIST_MAX)
-	{
-		MR_CFGERR_INVALID_FORMAT(instance->g_cfgfile_path, "service", "distmode");
-		return RT_ERR;
-	}
-
-	if (hashmode < 0 || distmode >= LDBC_HASH_MAX)
-	{
-		MR_CFGERR_INVALID_FORMAT(instance->g_cfgfile_path, "service", "hashmode");
-		return RT_ERR;
-	}
-
-	instance->dist_object = distributer_create(distmode, hashmode, 0);
-	if (instance->dist_object == NULL)
-	{
-		MR_ERROR("Create distributer handler failed. ");
-		return RT_ERR;
-	}
-
-	return RT_SUCCESS;
+    unsigned int distmode = LDBC_DIST_OUTER_TUPLE2;
+    unsigned int hashmode = LDBC_HASH_SYM_CRC;
+
+    MESA_load_profile_uint_def(instance->g_cfgfile_path, "service", "distmode",
+        &distmode, LDBC_DIST_OUTER_TUPLE2);
+    MESA_load_profile_uint_def(instance->g_cfgfile_path, "service", "hashmode",
+        &hashmode, LDBC_HASH_SYM_CRC);
+
+    if (distmode < 0 || distmode >= LDBC_DIST_MAX)
+    {
+        MR_CFGERR_INVALID_FORMAT(instance->g_cfgfile_path, "service", "distmode");
+        return RT_ERR;
+    }
+
+    if (hashmode < 0 || distmode >= LDBC_HASH_MAX)
+    {
+        MR_CFGERR_INVALID_FORMAT(instance->g_cfgfile_path, "service", "hashmode");
+        return RT_ERR;
+    }
+
+    instance->dist_object = distributer_create(distmode, hashmode, 0);
+    if (instance->dist_object == NULL)
+    {
+        MR_ERROR("Create distributer handler failed. ");
+        return RT_ERR;
+    }
+
+    return RT_SUCCESS;
 }
 
 static int mrapp_bpf_dumper_init(struct mr_instance * instance, struct mr_vdev * vdev)
 {
-	const char * str_devsym = vdev->devsym;
-	const char * str_cfgfile = instance->app_cfgfile_path;
-	
-	/* 从应用配置文件中读取Dumper的配置。Dumper系调试使用，由用户通过配置文件指定，
-	   读不到相关选项，则不启用Dumper。 */
-
-	char str_section[MR_SYMBOL_MAX] = { 0 };
-	snprintf(str_section, sizeof(str_section), "bpfdump:%s", str_devsym);
-	
-	unsigned int __opt_enable = 0;
-	MESA_load_profile_uint_def(str_cfgfile, str_section, "enable", &__opt_enable, 0);
-
-	/* 是否启用 */
-	if (!__opt_enable) return RT_SUCCESS;
-
-	char __opt_str_bpf_expr[MR_STRING_MAX] = { 0 };
-	char __opt_str_dumpfile[MR_STRING_MAX] = { 0 };
-
-	unsigned int __opt_direction = 0;
-	unsigned int __opt_backend = 0;
-	unsigned int __opt_offset = 0;
-
-	MESA_load_profile_string_def(str_cfgfile, str_section, "bpf_expr", __opt_str_bpf_expr,
-		sizeof(__opt_str_bpf_expr), NULL);
-	MESA_load_profile_string_def(str_cfgfile, str_section, "dumpfile", __opt_str_dumpfile,
-		sizeof(__opt_str_dumpfile), NULL);
-
-	MESA_load_profile_uint_def(str_cfgfile, str_section, "bpf_offset", &__opt_offset, 0);
-	MESA_load_profile_uint_def(str_cfgfile, str_section, "direction", &__opt_direction, 0);
-	MESA_load_profile_uint_def(str_cfgfile, str_section, "backend", &__opt_backend, 0);
-
-	vdev->bpf_dumper = bpf_dumper_create(instance->appsym, __opt_backend,
-		str_devsym, __opt_str_dumpfile, __opt_str_bpf_expr);
-
-	if (unlikely(vdev->bpf_dumper == NULL))
-	{
-		MR_ERROR("In application %s, BPF dumper for device %s create failed.",
-			instance->appsym, str_devsym); return RT_ERR;
-	}
-
-	return RT_SUCCESS;
+    const char * str_devsym = vdev->devsym;
+    const char * str_cfgfile = instance->app_cfgfile_path;
+
+    /* 从应用配置文件中读取Dumper的配置。Dumper系调试使用，由用户通过配置文件指定，
+       读不到相关选项，则不启用Dumper。 */
+
+    char str_section[MR_SYMBOL_MAX] = { 0 };
+    snprintf(str_section, sizeof(str_section), "bpfdump:%s", str_devsym);
+
+    unsigned int __opt_enable = 0;
+    MESA_load_profile_uint_def(str_cfgfile, str_section, "enable", &__opt_enable, 0);
+
+    /* 是否启用 */
+    if (!__opt_enable) return RT_SUCCESS;
+
+    char __opt_str_bpf_expr[MR_STRING_MAX] = { 0 };
+    char __opt_str_dumpfile[MR_STRING_MAX] = { 0 };
+
+    unsigned int __opt_direction = 0;
+    unsigned int __opt_backend = 0;
+    unsigned int __opt_offset = 0;
+
+    MESA_load_profile_string_def(str_cfgfile, str_section, "bpf_expr", __opt_str_bpf_expr,
+        sizeof(__opt_str_bpf_expr), NULL);
+    MESA_load_profile_string_def(str_cfgfile, str_section, "dumpfile", __opt_str_dumpfile,
+        sizeof(__opt_str_dumpfile), NULL);
+
+    MESA_load_profile_uint_def(str_cfgfile, str_section, "bpf_offset", &__opt_offset, 0);
+    MESA_load_profile_uint_def(str_cfgfile, str_section, "direction", &__opt_direction, 0);
+    MESA_load_profile_uint_def(str_cfgfile, str_section, "backend", &__opt_backend, 0);
+
+    vdev->bpf_dumper = bpf_dumper_create(instance->appsym, __opt_backend,
+        str_devsym, __opt_str_dumpfile, __opt_str_bpf_expr);
+
+    if (unlikely(vdev->bpf_dumper == NULL))
+    {
+        MR_ERROR("In application %s, BPF dumper for device %s create failed.",
+            instance->appsym, str_devsym); return RT_ERR;
+    }
+
+    return RT_SUCCESS;
 }
 
 /* 注册应用 */
@@ -227,9 +238,9 @@ static int mrapp_app_register(struct mr_instance * instance)
     struct ctrl_msg_app_reg_request reg_cmd;
     memset(&reg_cmd, 0, sizeof(reg_cmd));
 
-    ctrl_msg_header_construct(&reg_cmd.msg_header, 
-        sizeof(reg_cmd), 
-        CTRL_MSG_TYPE_REQUEST, 
+    ctrl_msg_header_construct(&reg_cmd.msg_header,
+        sizeof(reg_cmd),
+        CTRL_MSG_TYPE_REQUEST,
         CTRLMSG_TOPIC_APP_REGISTER);
 
     /* 应用标识符 */
@@ -285,16 +296,16 @@ static int mrapp_gconf_init(struct mr_instance * instance)
 
     char * g_cfg_file = cJSON_GetObjectItem(j_genernal, "g_cfgfile")->valuestring;
     if (g_cfg_file == NULL) goto j_parse_error;
-    
-	/* 全局配置文件路径 */
+
+    /* 全局配置文件路径 */
     strncpy(instance->g_cfgfile_path, g_cfg_file, sizeof(instance->g_cfgfile_path));
 
-	/* 本地配置文件路径 */
-	char __str_cfgfile[MR_STRING_MAX] = { 0 };
-	strncpy(__str_cfgfile, instance->g_cfgfile_path, sizeof(__str_cfgfile));
+    /* 本地配置文件路径 */
+    char __str_cfgfile[MR_STRING_MAX] = { 0 };
+    strncpy(__str_cfgfile, instance->g_cfgfile_path, sizeof(__str_cfgfile));
 
-	snprintf(instance->app_cfgfile_path, sizeof(instance->app_cfgfile_path),
-		"%s/mrapp.%s.conf", dirname(__str_cfgfile), instance->appsym);
+    snprintf(instance->app_cfgfile_path, sizeof(instance->app_cfgfile_path),
+        "%s/mrapp.%s.conf", dirname(__str_cfgfile), instance->appsym);
 
     return RT_SUCCESS;
 
@@ -307,14 +318,14 @@ extern int mrapp_monit_loop(struct mr_instance * instance);
 
 void * mrapp_ctrlplane_thread(void * args)
 {
-	struct mr_instance * _instance = (struct mr_instance *)args;
-	pthread_detach(pthread_self());
-
-	while (1)
-	{
-		mrapp_monit_loop(_instance);
-		sleep(1);
-	}
+    struct mr_instance * _instance = (struct mr_instance *)args;
+    pthread_detach(pthread_self());
+
+    while (1)
+    {
+        mrapp_monit_loop(_instance);
+        sleep(1);
+    }
 }
 
 struct mr_vdev * marsio_device_lookup(struct mr_instance * instance, const char * devsym)
@@ -335,7 +346,7 @@ static int __ctrlplane_conn_close_handler(struct ctrlmsg_handler * ct_hand,
     exit(EXIT_FAILURE);
 }
 
-static int __open_device_response_handler(struct ctrlmsg_handler * ct_hand, 
+static int __open_device_response_handler(struct ctrlmsg_handler * ct_hand,
     struct ctrlmsg_conn * ct_conn, struct ctrl_msg_header * msg, void * arg)
 {
     struct ctrl_msg_vdev_open_response * rep_msg = (struct ctrl_msg_vdev_open_response *)msg;
@@ -366,7 +377,7 @@ static int __open_device_response_handler(struct ctrlmsg_handler * ct_hand,
     }
 
     instance->nr_vdevs++;
-    
+
 wake_up:
     pthread_mutex_lock(&instance->lock_ctrlmsg_wait);
     instance->ctrlmsg_wait = 1;
@@ -396,12 +407,12 @@ static int __app_register_response_handler(struct ctrlmsg_handler * ct_hand,
     return 0;
 }
 
-struct mr_vdev * marsio_open_device(struct mr_instance * instance, 
+struct mr_vdev * marsio_open_device(struct mr_instance * instance,
     const char * devsym, unsigned int nr_rxstream, unsigned int nr_txstream)
 {
     /* 构造虚设备打开请求 */
     struct ctrl_msg_vdev_open_request req_msg;
-    ctrl_msg_header_construct(&req_msg.msg_header, sizeof(req_msg), 
+    ctrl_msg_header_construct(&req_msg.msg_header, sizeof(req_msg),
         CTRL_MSG_TYPE_REQUEST, CTRLMSG_TOPIC_VDEV_OPEN);
 
     snprintf((char *)req_msg.devsym, sizeof(req_msg.devsym), "%s", devsym);
@@ -433,8 +444,8 @@ struct mr_vdev * marsio_open_device(struct mr_instance * instance,
     MR_INFO("   ARP protocol handler           : %s", vdev->en_arp ? "Enable" : "Disable");
     MR_INFO("   ICMP protocol handler          : %s", vdev->en_arp ? "Enable" : "Disable");
 
-	/* 调试捕包 */
-	mrapp_bpf_dumper_init(instance, vdev);
+    /* 调试捕包 */
+    mrapp_bpf_dumper_init(instance, vdev);
     return vdev;
 }
 
@@ -499,7 +510,7 @@ struct mr_instance * marsio_current()
 }
 
 int marsio_init(struct mr_instance * instance, const char * appsym)
-{    
+{
     /* 写应用名称参数 */
     snprintf(instance->appsym, sizeof(instance->appsym), "%s", appsym);
 
@@ -558,12 +569,12 @@ int marsio_init(struct mr_instance * instance, const char * appsym)
         goto err;
     }
 
-	/* 负载均衡器 */
-	if (mrapp_distributer_init(instance) != RT_SUCCESS)
-	{
-		MR_ERROR("Distributer initialization failed.");
-		goto err;
-	}
+    /* 负载均衡器 */
+    if (mrapp_distributer_init(instance) != RT_SUCCESS)
+    {
+        MR_ERROR("Distributer initialization failed.");
+        goto err;
+    }
 
     instance->neigh = malloc(sizeof(struct neighbour_manager));
     MR_VERIFY_MALLOC(instance->neigh);
@@ -647,5 +658,5 @@ out:
 
 int marsio_destory(struct mr_instance * instance)
 {
-	return 0;
+    return 0;
 }
\ No newline at end of file