整合协议栈模块与service模块，对接接口。

14 files changed, 160 insertions, 78 deletions

diff --git a/service/include/sc_common.h b/service/include/sc_common.h
index de26a20..bf45036 100644
--- a/service/include/sc_common.h
+++ b/service/include/sc_common.h
@@ -21,6 +21,8 @@ struct sc_device
 	unsigned int en_mr_dev;
 	/* �������� */
 	struct mr_dev_param mr_dev_param;
+	/* Э��ջ�豸��� */
+	struct sk_dev_desc * sk_dev;
 	/* Э��ջ�豸ʹ�� */
 	unsigned int en_sk_dev;
 	/* Э��ջ�豸���� */
diff --git a/service/src/register.c b/service/src/register.c
index a75f867..c78eb17 100644
--- a/service/src/register.c
+++ b/service/src/register.c
@@ -113,6 +113,9 @@ static int __rt_dev_register(struct sc_instance * instance,
 static int __sk_dev_register(struct sc_instance * instance,
 	struct sc_device * sc_dev)
 {
+	// 计算默认队列数，队列数等于IO核心数
+	int nr_queues = mask_popcnt(instance->cpu_mask);
+
 	// 跳过没有启用协议栈的设备
 	if(!(sc_dev->en_mr_dev && sc_dev->en_sk_dev))
 	{
@@ -127,14 +130,14 @@ static int __sk_dev_register(struct sc_instance * instance,
 	// TODO:错误处理
 	assert(direct_pool != NULL && indirect_pool != NULL);
 	
-	int ret = sk_device_create(instance->sk_instance, &sc_dev->sk_dev_param,
-		direct_pool, indirect_pool);
+	sc_dev->sk_dev = sk_device_create(instance->sk_instance, &sc_dev->sk_dev_param,
+	                                  direct_pool, indirect_pool, nr_queues);
 	
-	if(unlikely(ret < 0))
+	if(unlikely(sc_dev->sk_dev == NULL))
 	{
 		MR_LOG(WARNING, SERVICE, "Service, StackDeviceRegister, "
 			"StackDevice %s create failed. \n", sc_dev->sk_dev_param.symbol);
-		return ret;
+		return -1;
 	}
 
 	MR_LOG(WARNING, SERVICE, "Service, StackDeviceRegister, "
diff --git a/service/src/runtime.c b/service/src/runtime.c
index b713189..28887a7 100644
--- a/service/src/runtime.c
+++ b/service/src/runtime.c
@@ -66,6 +66,7 @@ void sc_thread_device_attach(struct sc_instance * instance,
 		tinstance->rt_dev[nr_devices].rx_devq = rx_devq;
 		tinstance->rt_dev[nr_devices].tx_devq = tx_devq;
 		tinstance->rt_dev[nr_devices].rtstream = rtstream;
+		tinstance->rt_dev[nr_devices].sk_dev = sc_dev_iter->sk_dev;
 		nr_devices++;
 	}
 
diff --git a/service/src/rxtx.c b/service/src/rxtx.c
index 76d7a84..bcba6e1 100644
--- a/service/src/rxtx.c
+++ b/service/src/rxtx.c
@@ -19,7 +19,7 @@ static inline void __rx_loop_one_device(struct sc_instance * instance,
 	struct sc_thread_instance * tinstance, struct sc_rt_device * rtdev,
 	unsigned int nr_rx_burst)
 {
-	struct rte_mbuf * rx_bufs[MR_BURST_MAX];
+	struct rte_mbuf * rx_bufs_in[MR_BURST_MAX];
 	hash_t hash_result[MR_BURST_MAX];
 	int ret;
 
@@ -27,17 +27,19 @@ static inline void __rx_loop_one_device(struct sc_instance * instance,
 	struct mr_rtdev_stream * rtstream = rtdev->rtstream;
 
 	uint16_t nr_rx_bufs = rte_eth_rx_burst(devq->port_id,
-		devq->queue_id, rx_bufs, nr_rx_burst);
+		devq->queue_id, rx_bufs_in, nr_rx_burst);
 
 	// û�����ݰ�ֱ������
 	if (unlikely(nr_rx_bufs == 0)) return;
 
-	int nr_rx_bufs_len = pktslen(rx_bufs, nr_rx_bufs);
+	int nr_rx_bufs_len = pktslen(rx_bufs_in, nr_rx_bufs);
 	UPDATE_DEV_STAT(dev_rx_frame, devq->port_id, nr_rx_bufs);
 	UPDATE_DEV_STAT(dev_rx_bit, devq->port_id, nr_rx_bufs_len);
 	UPDATE_APP_STAT(rx_frame, nr_rx_bufs);
 	UPDATE_APP_STAT(rx_byte, nr_rx_bufs_len);
 
+
+#if 0
 	// �������ݰ����طֵ�ʹ�õ�Hashֵ
 	ret = distributer_hash_caculate(instance->distmode,
 		rx_bufs, nr_rx_bufs, hash_result);
@@ -48,11 +50,25 @@ static inline void __rx_loop_one_device(struct sc_instance * instance,
 	ret = vnode_enqueue_burst_with_hash(attach_desc->prod, &vnodeman_ops,
 		attach_desc->queue_id, (void **)rx_bufs, hash_result, nr_rx_bufs);
 	if (unlikely(ret < 0)) goto err_free_mbufs;
+
+#endif
+
+	thread_id_t sid = mr_thread_id();
+
+	struct rte_mbuf * rx_bufs_out[MR_BURST_MAX];
+	int nr_rx_bufs_out;
+	
+	sk_serv_device_rx_bulk(rtdev->sk_dev, sid, rx_bufs_in, rx_bufs_out,
+		nr_rx_bufs, &nr_rx_bufs_out);
+	
+	for (int i = 0; i < nr_rx_bufs_out; i++)
+		rte_pktmbuf_free(rx_bufs_out[i]);
+	
 	return;
 
 err_free_mbufs:
 	for (int i = 0; i < nr_rx_bufs; i++)
-		rte_pktmbuf_free(rx_bufs[i]);
+		rte_pktmbuf_free(rx_bufs_in[i]);
 
 	UPDATE_APP_STAT(drop_frame, nr_rx_bufs);
 	return;
diff --git a/stack/CMakeLists.txt b/stack/CMakeLists.txt
index ab0d708..345e8b4 100644
--- a/stack/CMakeLists.txt
+++ b/stack/CMakeLists.txt
@@ -6,8 +6,8 @@ include_directories(${DPDK_INCLUDE_DIR})
 

 add_definitions(${DPDK_C_PREDEFINED})

 include_directories(include)

-add_library(stack src/stack.c src/device.c src/neigh.c src/protocol-common.c

-	src/protocol-arp.c src/protocol-raw.c)

+add_library(stack src/stack.c src/device.c src/neigh.c src/rxtx.c src/protocol-common.c

+	src/protocol-arp.c src/protocol-tcp.c src/protocol-udp.c src/protocol-raw.c)

 target_link_libraries(stack MESA_prof_load_static mruntime core)

 target_link_libraries(stack rt pthread dl)

 target_include_directories(stack INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include/")
\ No newline at end of file
diff --git a/stack/include/sk_device.h b/stack/include/sk_device.h
index b251a25..a06947e 100644
--- a/stack/include/sk_device.h
+++ b/stack/include/sk_device.h
@@ -89,6 +89,18 @@ struct sk_dev_info
 	struct ether_addr mac_addr;
 };
 
+enum sk_dev_sid_type
+{
+	SK_DEVQ_TYPE_SERV,
+	SK_DEVQ_TYPE_RAW_RX,
+	SK_DEVQ_TYPE_RAW_TX,
+	SK_DEVQ_TYPE_TCP_RX,
+	SK_DEVQ_TYPE_TCP_TX,
+	SK_DEVQ_TYPE_UDP_RX,
+	SK_DEVQ_TYPE_UDP_TX,
+	SK_DEVQ_TYPE_MAX
+};
+
 // �豸��������ӽ���ͨ�ã�
 struct sk_dev_desc
 {
@@ -124,6 +136,13 @@ struct sk_dev_desc
 	struct mr_rtdev_stream * udp_rtdev_rxstream[MR_SID_MAX];
 	// UDP�� Rt-Device Stream
 	struct mr_rtdev_stream * udp_rtdev_txstream[MR_SID_MAX];
+
+	// SID��������
+	rte_spinlock_t sid_alloc_lock;
+	// SID������
+	uint8_t sid_alloc_record[SK_DEVQ_TYPE_MAX][MR_SID_MAX];
+	// SID������¼
+	uint8_t nr_sid[SK_DEVQ_TYPE_MAX];
 };
 
 TAILQ_HEAD(sk_dev_info_list, sk_dev_info);
@@ -140,12 +159,18 @@ struct sk_dev_desc * sk_device_desc_lookup(struct sk_dev_desc_list * desc_list,
 int sk_device_iterate(struct sk_instance * instance, struct sk_dev_info ** dev_info);
 
 // ����һ��Э��ջ�豸
-int sk_device_create(struct sk_instance * instance, struct sk_dev_param * param, 
-	struct rte_mempool* direct_pool, struct rte_mempool* indirect_pool);
+struct sk_dev_desc * sk_device_create(struct sk_instance* instance, struct sk_dev_param* param,
+	struct rte_mempool* direct_pool, struct rte_mempool* indirect_pool, unsigned nr_serv_thread);
 
 // ����Э��ջ�豸
 int sk_device_destory(struct sk_dev_info * devinfo);
 
+int sk_device_attach(struct sk_dev_desc * dev_desc, enum sk_dev_sid_type type,
+	thread_id_t * out_sid);
+
+void sk_device_deattach(struct sk_dev_desc * dev_desc, enum sk_dev_sid_type type,
+	thread_id_t sid);
+
 // ����Э��ջ��IP��ַ
 int sk_device_set_inaddr(struct sk_dev_info * dev_info, struct in_addr in_addr,
 	struct in_addr in_mask);
diff --git a/stack/include/sk_protocol_arp.h b/stack/include/sk_protocol_arp.h
index 9ca93e9..6c3adfd 100644
--- a/stack/include/sk_protocol_arp.h
+++ b/stack/include/sk_protocol_arp.h
@@ -1,5 +1,8 @@
 #pragma once

 

+#include <rte_mbuf.h>

+#include <sk_device.h>

+

 void protocol_serv_arp_entry(struct sk_dev_desc* dev_desc,

 	thread_id_t sid, struct rte_mbuf* mbufs_in[], int nr_mbufs_in);

 

diff --git a/stack/include/sk_protocol_common.h b/stack/include/sk_protocol_common.h
index 584e538..8a02ac4 100644
--- a/stack/include/sk_protocol_common.h
+++ b/stack/include/sk_protocol_common.h
@@ -1,8 +1,6 @@
 #pragma once

 

 #include <mr_rtdev.h>

-#include <sk_common.h>

-#include <sk_stack.h>

 #include <sk_device.h>

 

 // ����Э��ӽ��̳�ʼ�����������ٶ�Ӧ������ʱ�豸��

diff --git a/stack/include/sk_protocol_tcp.h b/stack/include/sk_protocol_tcp.h
index fb24bb2..9db4f2b 100644
--- a/stack/include/sk_protocol_tcp.h
+++ b/stack/include/sk_protocol_tcp.h
@@ -1,4 +1,7 @@
 #pragma once

 

+#include <rte_mbuf.h>

+#include <sk_device.h>

+

 void protocol_serv_tcp_entry(struct sk_dev_desc* dev_desc, thread_id_t sid, 

 	struct rte_mbuf* mbufs_in[], int nr_mbufs_in);
\ No newline at end of file
diff --git a/stack/include/sk_protocol_udp.h b/stack/include/sk_protocol_udp.h
index c416278..50c7c1f 100644
--- a/stack/include/sk_protocol_udp.h
+++ b/stack/include/sk_protocol_udp.h
@@ -1,4 +1,7 @@
 #pragma once

 

+#include <rte_mbuf.h>

+#include <sk_device.h>

+

 void protocol_serv_udp_entry(struct sk_dev_desc* dev_desc,

 	thread_id_t sid, struct rte_mbuf* mbufs_in[], int nr_mbufs_in);
\ No newline at end of file
diff --git a/stack/src/device.c b/stack/src/device.c
index 2512f62..15f568c 100644
--- a/stack/src/device.c
+++ b/stack/src/device.c
@@ -137,8 +137,8 @@ err_out:
 
 
 // 创建一个协议栈设备
-int sk_device_create(struct sk_instance * instance, struct sk_dev_param * param, 
-	struct rte_mempool * direct_pool, struct rte_mempool * indirect_pool)
+struct sk_dev_desc * sk_device_create(struct sk_instance* instance, struct sk_dev_param* param,
+	struct rte_mempool* direct_pool, struct rte_mempool* indirect_pool, unsigned nr_serv_thread)
 {
 	// 检查设备是否已经存在，不允许重复创建
 	struct sk_dev_info * dev_info = sk_device_lookup(instance, param->symbol);
@@ -146,7 +146,7 @@ int sk_device_create(struct sk_instance * instance, struct sk_dev_param * param,
 	{
 		MR_LOG(INFO, STACK, "StackCreateDevice, StackDevice %s has been created. failed. \n",
 			param->symbol);
-		return -EEXIST;
+		goto err_out;
 	}
 
 	// 申请Info结构体的空间
@@ -154,7 +154,7 @@ int sk_device_create(struct sk_instance * instance, struct sk_dev_param * param,
 	if(unlikely(dev_info == NULL))
 	{
 		MR_LOG(WARNING, STACK, "StackCreateDevice, Cannot alloc memory for device info.\n");
-		return -ENOMEM;
+		goto err_out;
 	}
 
 	// 写参数
@@ -194,12 +194,12 @@ int sk_device_create(struct sk_instance * instance, struct sk_dev_param * param,
 	TAILQ_INSERT_TAIL(&instance->dev_desc_list, dev_desc, next);
 	// TODO: 详细的日志输出
 	MR_LOG(NOTICE, STACK, "Sk-device %s created. \n", dev_info->symbol);
-	return 0;
+	return dev_desc;
 
 err_out:
 	MR_LOG(ERR, STACK, "Sk-device %s create failed. \n", dev_info->symbol);
 	if (dev_info) rte_free(dev_info);
-	return 0;
+	return NULL;
 }
 
 // 打开一个协议栈设备（在从进程）
@@ -238,6 +238,30 @@ err_out:
 	return -1;
 }
 
+int sk_device_attach(struct sk_dev_desc * dev_desc, enum sk_dev_sid_type type,
+	thread_id_t * out_sid)
+{
+	rte_spinlock_lock(&dev_desc->sid_alloc_lock);
+	int ret = -1;
+	for(int i = 0; i < dev_desc->nr_sid[type]; i++)
+	{
+		if (dev_desc->sid_alloc_record[type][i]) continue;
+		dev_desc->sid_alloc_record[type][i] = 1;
+		*out_sid = i;
+		ret = 0; break;
+	}
+	rte_spinlock_lock(&dev_desc->sid_alloc_lock);
+	return ret;
+}
+
+void sk_device_deattach(struct sk_dev_desc * dev_desc, enum sk_dev_sid_type type,
+	thread_id_t sid)
+{
+	rte_spinlock_lock(&dev_desc->sid_alloc_lock);
+	dev_desc->sid_alloc_record[type][sid] = 0;
+	rte_spinlock_lock(&dev_desc->sid_alloc_lock);
+}
+
 
 // 销毁协议栈设备
 int sk_device_destory(struct sk_dev_info * devinfo)
diff --git a/stack/src/protocol-tcp.c b/stack/src/protocol-tcp.c
index 0af46d8..138e81d 100644
--- a/stack/src/protocol-tcp.c
+++ b/stack/src/protocol-tcp.c
@@ -5,4 +5,4 @@ void protocol_serv_tcp_entry(struct sk_dev_desc* dev_desc,
 	thread_id_t sid, struct rte_mbuf* mbufs_in[], int nr_mbufs_in)

 {

 	return;

-}

+}
\ No newline at end of file
diff --git a/stack/src/rxtx.c b/stack/src/rxtx.c
index 92fed52..b3905a0 100644
--- a/stack/src/rxtx.c
+++ b/stack/src/rxtx.c
@@ -65,6 +65,7 @@ int sk_serv_device_rx_bulk(struct sk_dev_desc * dev_desc, thread_id_t sid,
 	protocol_serv_arp_entry(dev_desc, sid, packets[__PACKET_TYPE_ARP].packets,

 		packets[__PACKET_TYPE_ARP].nr_packets);

 

+#if 0

 	// ICMP���Ĵ���

 	protocol_serv_icmp_entry(dev_desc, sid, packets[__PACKET_TYPE_ICMP].packets,

 		packets[__PACKET_TYPE_TCP].nr_packets);

@@ -77,6 +78,8 @@ int sk_serv_device_rx_bulk(struct sk_dev_desc * dev_desc, thread_id_t sid,
 	protocol_serv_udp_entry(dev_desc, sid, packets[__PACKET_TYPE_UDP].packets, 

 		packets[__PACKET_TYPE_UDP].nr_packets);

 

+#endif

+

 	// �޷������ı��ģ����ظ��ϲ�

 	for(int i = 0; i < packets[__PACKET_TYPE_ELSE].nr_packets; i++)

 	{

@@ -84,5 +87,6 @@ int sk_serv_device_rx_bulk(struct sk_dev_desc * dev_desc, thread_id_t sid,
 		assert(i <= nr_mbufs_in);

 	}

 	

+	*nr_mbufs_out = packets[__PACKET_TYPE_ELSE].nr_packets;

 	return 0;

 }
\ No newline at end of file
diff --git a/test/TestVMan.cc b/test/TestVMan.cc
index ca8ed11..3e30bad 100644
--- a/test/TestVMan.cc
+++ b/test/TestVMan.cc
@@ -1,59 +1,59 @@
-

-/* \brief ����ڵ��������Ԫ���Դ���

- * \author Qiuwen Lu<[email protected]>

- * \date 2016-09-02

- */

-

-#include <gtest/gtest.h>

-#include <mr_ctx.h>

-#include <mr_common.h>

-#include <mr_vman.h>

-

-extern struct mr_module module_desc_vnodeman;

-

-class TestCaseVNodeMan : public ::testing::Test

-{

-protected:

-

-	TestCaseVNodeMan()

-	{

-	}

-

-	~TestCaseVNodeMan()

-	{

-	}

-

-	static void SetUpTestCase()

-	{

-		mr_global_cfg_init();

-		mr_global_ctx_init();

-

+
+/* \brief ����ڵ��������Ԫ���Դ���
+ * \author Qiuwen Lu<[email protected]>
+ * \date 2016-09-02
+ */
+
+#include <gtest/gtest.h>
+#include <mr_ctx.h>
+#include <mr_common.h>
+#include <mr_vman.h>
+
+extern struct mr_module module_desc_vnodeman;
+
+class TestCaseVNodeMan : public ::testing::Test
+{
+protected:
+
+	TestCaseVNodeMan()
+	{
+	}
+
+	~TestCaseVNodeMan()
+	{
+	}
+
+	static void SetUpTestCase()
+	{
+		mr_global_cfg_init();
+		mr_global_ctx_init();
+
 		struct mr_global_ctx * g_ctx = mr_global_ctx_get();
-		struct mr_config * g_cfg = mr_global_config_get();

-
-		int ret = module_desc_vnodeman.preinit(

-			&module_desc_vnodeman, g_cfg, g_ctx);

-		ASSERT_EQ(ret, 0);

-	}

-};

-

-TEST_F(TestCaseVNodeMan, TestRegisterAndAttach)

-{

-	int ret = 0;

-	ret = mr_vnodeman_create("TN", 1024, 128, 0);

-	ASSERT_EQ(ret, 0);

-	ret = mr_vnodeman_register_consumer("TN", "TNS",4);

-	ASSERT_EQ(ret, 0);

-	ret = mr_vnodeman_register_producer("TN", "TNP", 4);

-	ASSERT_EQ(ret, 0);

-

-	struct vnodeman_attach_desc * cons_desc[4];

-	struct vnodeman_attach_desc * prod_desc[4];

-	for (int i = 0; i < 4; i++)

-	{

-		cons_desc[i] = mr_vnodeman_consumer_attach("TN", "TNS");

-		ASSERT_TRUE(cons_desc[i] != NULL);

-		EXPECT_EQ(cons_desc[i]->desc_type, VNODEMAN_ATTACH_CONSUMER);

-		EXPECT_EQ(cons_desc[i]->queue_id, i);

-	}

+		struct mr_config * g_cfg = mr_global_config_get();
+
+		int ret = module_desc_vnodeman.preinit(
+			&module_desc_vnodeman, g_cfg, g_ctx);
+		ASSERT_EQ(ret, 0);
+	}
+};
+
+TEST_F(TestCaseVNodeMan, TestRegisterAndAttach)
+{
+	int ret = 0;
+	ret = mr_vnodeman_create("TN", 1024, 128, 0);
+	ASSERT_EQ(ret, 0);
+	ret = mr_vnodeman_register_consumer("TN", "TNS",4);
+	ASSERT_EQ(ret, 0);
+	ret = mr_vnodeman_register_producer("TN", "TNP", 4);
+	ASSERT_EQ(ret, 0);
+
+	struct vnodeman_attach_desc * cons_desc[4];
+	struct vnodeman_attach_desc * prod_desc[4];
+	for (int i = 0; i < 4; i++)
+	{
+		cons_desc[i] = mr_vnodeman_consumer_attach("TN", "TNS");
+		ASSERT_TRUE(cons_desc[i] != NULL);
+		EXPECT_EQ(cons_desc[i]->desc_type, VNODEMAN_ATTACH_CONSUMER);
+		EXPECT_EQ(cons_desc[i]->queue_id, i);
+	}
 }
\ No newline at end of file