重构Sendpath部分，增加多种Sendpath的统一接口。支持对Sendpath进行选项设置。重构快速发包路径。

1 files changed, 253 insertions, 62 deletions

diff --git a/app/src/sendpath.c b/app/src/sendpath.c
index b98aa93..0b1f021 100644
--- a/app/src/sendpath.c
+++ b/app/src/sendpath.c
@@ -4,6 +4,8 @@
 #include <sendpath.h>
 #include <rte_ip.h>
 #include <rte_ether.h>
+#include <assert.h>
+#include <arpa/inet.h>
 
 static inline int is_same_subnet(struct in_addr a_addr, struct in_addr b_addr, struct in_addr mask)
 {
@@ -13,33 +15,70 @@ static inline int is_same_subnet(struct in_addr a_addr, struct in_addr b_addr, s
     return (in_addr_a & in_addr_mask) == (in_addr_b & in_addr_mask);
 }
 
-static int droute_l2_construct_fn(struct mr_sendpath * sendpath,
+/* ======================================================================================= */
+
+static void sendpath_vdev_destory(struct mr_sendpath * sendpath)
+{
+    struct __mr_sendpath_vdev * _sendpath = container_of(sendpath, struct __mr_sendpath_vdev, _father);
+    free(_sendpath);
+}
+
+static struct mr_sendpath * sendpath_vdev_create(struct mr_instance * instance, int type, va_list va_list)
+{
+    /* 第一个传入参数：目标虚设备句柄 */
+    struct mr_vdev * dest_vdev = va_arg(va_list, struct mr_vdev *);
+
+    struct __mr_sendpath_vdev * _sendpath = malloc(sizeof(struct __mr_sendpath_vdev));
+    memset(_sendpath, 0, sizeof(struct __mr_sendpath_vdev));
+
+    _sendpath->_father.fn_destory = sendpath_vdev_destory;
+    _sendpath->_father.target_vdi = dest_vdev->vdi;
+    _sendpath->_father.can_use = 1;
+    return &_sendpath->_father;
+}
+
+/* ======================================================================================= */
+
+static void sendpath_route_destory(struct mr_sendpath * sendpath)
+{
+    struct __mr_sendpath_route * _sendpath = container_of(sendpath,
+        struct __mr_sendpath_route, _father);
+
+    free(_sendpath);
+}
+
+static int sendpath_route_l2_construct(struct mr_sendpath * sendpath,
     struct rte_mbuf * mbuf[], unsigned int nr_mbuf)
 {
+    struct __mr_sendpath_route * _sendpath = container_of(sendpath,
+        struct __mr_sendpath_route, _father);
+
     for (int i = 0; i < nr_mbuf; i++)
     {
         struct ether_hdr * ether_hdr = (struct ether_hdr *)rte_pktmbuf_prepend(mbuf[i],
             sizeof(struct ether_hdr));
 
         MR_VERIFY_2(ether_hdr != NULL, "Not enough space for ethernet header in mbufs. ");
-        ether_hdr->s_addr = sendpath->src_eth_addr;
-        ether_hdr->d_addr = sendpath->dst_eth_addr;
+        ether_hdr->s_addr = _sendpath->src_eth_addr;
+        ether_hdr->d_addr = _sendpath->dst_eth_addr;
         ether_hdr->ether_type = htons(ETHER_TYPE_IPv4);
     }
 
     return 0;
 }
 
-static int droute_l3_construct_fn(struct mr_sendpath * sendpath,
+static int sendpath_route_l3_construct(struct mr_sendpath * sendpath,
     struct rte_mbuf * mbuf[], unsigned int nr_mbuf)
 {
+    struct __mr_sendpath_route * _sendpath = container_of(sendpath,
+        struct __mr_sendpath_route, _father);
+
     for (int i = 0; i < nr_mbuf; i++)
     {
         struct ipv4_hdr * ip_hdr = rte_pktmbuf_mtod(mbuf[i], struct ipv4_hdr *);
-        ip_hdr->src_addr = sendpath->src_addr.s_addr;
-        ip_hdr->dst_addr = sendpath->dst_addr.s_addr;
+        ip_hdr->src_addr = _sendpath->src_addr.s_addr;
+        ip_hdr->dst_addr = _sendpath->dst_addr.s_addr;
 
-        /* 重新计算校验和 */
         ip_hdr->hdr_checksum = 0;
         ip_hdr->hdr_checksum = rte_ipv4_cksum(ip_hdr);
     }
@@ -47,112 +86,264 @@ static int droute_l3_construct_fn(struct mr_sendpath * sendpath,
     return 0;
 }
 
-static int droute_requery(struct mr_sendpath * sendpath)
+static int sendpath_route_requery(struct mr_sendpath * sendpath)
 {
     struct mr_instance * instance = sendpath->instance;
-    int ret = neigh_query(instance->neigh, sendpath->target_addr,
-        &sendpath->dst_eth_addr, &sendpath->target_vdi);
+    struct __mr_sendpath_route * _sendpath = container_of(sendpath,
+        struct __mr_sendpath_route, _father);
+
+    int ret = neigh_query(instance->neigh, _sendpath->target_addr,
+        &_sendpath->dst_eth_addr, &sendpath->target_vdi);
 
     if (ret < 0)
     {
-        neigh_create_or_update(instance->neigh, sendpath->target_addr,
-            NULL, sendpath->target_vdi, 0);
-        return RT_ERR;
+        neigh_create_or_update(instance->neigh, _sendpath->target_addr, NULL,
+            sendpath->target_vdi, 0); return RT_ERR;
     }
     else
     {
         sendpath->can_use = 1;
         return RT_SUCCESS;
     }
+
+    return RT_SUCCESS;
 }
 
-struct mr_sendpath * marsio_sendpath_create_by_droute(struct mr_vdev * dest_device, struct in_addr addr)
+static int sendpath_route_option_set(struct mr_instance * instance, struct mr_sendpath * sendpath, 
+    int opt, va_list va_list)
 {
-    struct vdev * vdev = dest_device->vdi->vdev;
-    struct in_addr target_in_addr;
+    if (opt == MR_SENDPATH_OPT_BUILD_L2)
+    {
+        unsigned int enable = va_arg(va_list, unsigned int);
+        sendpath->fn_l2_construct = enable ? sendpath_route_l2_construct : NULL;
+    }
+    else if (opt == MR_SENDPATH_OPT_BUILD_L3)
+    {
+        unsigned int enable = va_arg(va_list, unsigned int);
+        sendpath->fn_l3_construct = enable ? sendpath_route_l3_construct : NULL;
+    }
+    else
+    {
+        MR_ERROR("Invalided opt type in %s()", __FUNCTION__);
+        return -EINVAL;
+    }
 
-    /* 同一网段，查询目标IP对应的MAC地址，不同网段，查询网关的MAC地址 */
-    if (is_same_subnet(addr, vdev->in_addr, vdev->in_mask))
+    return 0;
+}
+
+static int sendpath_route_option_get(struct mr_instance * instance, struct mr_sendpath * sendpath, 
+    int opt, va_list va_list)
+{
+    return 0;
+}
+
+static struct mr_sendpath * sendpath_route_create(struct mr_instance * instance, int type, va_list va_list)
+{
+    struct mr_vdev * target_vdev;
+    struct in_addr in_addr;
+
+    /* 对于一般的路由，传入目标地址 */
+    if (type == MR_SENDPATH_ROUTE_NORMAL)
+    {
+        target_vdev = NULL;
+        in_addr = va_arg(va_list, struct in_addr);
+    }
+
+    /* 对于指定出口设备的路由，传入（1）设备句柄（2）目标IP地址 */
+    else if (type == MR_SENDPATH_ROUTE_SPEC_DEV)
     {
-        target_in_addr = addr;
+        target_vdev = va_arg(va_list, struct mr_vdev *);
+        in_addr = va_arg(va_list, struct in_addr);
     }
     else
     {
-        target_in_addr = vdev->in_gateway;
+        assert(0);
     }
 
-    MR_DEBUG("SENDPATH: create_by_droute, in_addr=%u, target_in_addr=%u",
-        addr.s_addr, target_in_addr.s_addr);
-    
-    struct mr_sendpath * sendpath = malloc(sizeof(struct mr_sendpath));
-    MR_VERIFY_MALLOC(sendpath);
-
-    memset(sendpath, 0, sizeof(struct mr_sendpath));
-    sendpath->instance = dest_device->instance;
-    sendpath->fn_requery = droute_requery;
-    sendpath->fn_l2_construct = droute_l2_construct_fn;
-    sendpath->fn_l3_construct = droute_l3_construct_fn;
+    struct in_addr target_in_addr = { INADDR_NONE };
+
+    /* 查找合适的设备 */
+    if (target_vdev != NULL) goto _build;
+
+    for (int i = 0; i < instance->nr_vdevs; i++)
+    {
+        struct mr_vdev * vdev = &instance->vdevs[i];
+        struct vdev * __vdev = vdev->vdi->vdev;
+
+        if (!is_same_subnet(in_addr, __vdev->in_addr, __vdev->in_mask))
+            continue;
+
+        target_in_addr = in_addr;
+        target_vdev = vdev;
+    }
+
+_build:
+
+    /* 没有找到设备，说明没有路由 */
+    if (target_vdev == NULL)
+    {
+        char str_in_addr[MR_STRING_MAX] = { 0 };
+        inet_ntop(AF_INET, &in_addr, str_in_addr, INET_ADDRSTRLEN);
+
+        MR_ERROR("No route to address %s, creating route sendpath failed. ",
+            str_in_addr); return NULL;
+    }
+
+    struct vdev * __target_vdev = target_vdev->vdi->vdev;
+
+    /* 判断目标IP地址是否属于网卡所在的网段，如不在，走网关 */
+    if (is_same_subnet(in_addr, __target_vdev->in_addr, __target_vdev->in_mask))
+    {
+        target_in_addr = in_addr;
+    }
+    else
+    {
+        target_in_addr = __target_vdev->in_gateway;
+    }
+
+    /* 目的地址不合法，返回 */
+    if (target_in_addr.s_addr == htonl(INADDR_ANY) ||
+        target_in_addr.s_addr == htonl(INADDR_LOOPBACK) ||
+        target_in_addr.s_addr == htonl(INADDR_NONE))
+    {
+        char str_target_in_addr[MR_STRING_MAX] = { 0 };
+        char str_in_addr[MR_STRING_MAX] = { 0 };
+
+        inet_ntop(AF_INET, &target_in_addr, str_target_in_addr, INET_ADDRSTRLEN);
+        inet_ntop(AF_INET, &in_addr, str_in_addr, INET_ADDRSTRLEN);
+        
+        MR_WARNING("Invalid target ip address %s(or next hop address), "
+            "creating route sendpath for %s failed. ", str_target_in_addr,
+            str_in_addr); return NULL;
+    }
+
+    struct __mr_sendpath_route * _sendpath = malloc(sizeof(struct __mr_sendpath_route));
+    struct mr_sendpath * sendpath = &_sendpath->_father;
+
+    /* 填充SendPath各虚函数指针*/
+    sendpath->instance = target_vdev->instance;
+    sendpath->target_vdi = target_vdev->vdi;
+
+    sendpath->fn_requery = sendpath_route_requery;
+    sendpath->fn_l2_construct = sendpath_route_l2_construct;
+    sendpath->fn_l3_construct = sendpath_route_l3_construct;
     sendpath->fn_l4_construct = NULL;
-    sendpath->src_addr = vdev->in_addr;
-    sendpath->dst_addr = addr;
-    sendpath->target_vdi = dest_device->vdi;
+
+    sendpath->fn_option_get = sendpath_route_option_get;
+    sendpath->fn_option_set = sendpath_route_option_set; 
+    sendpath->fn_destory = sendpath_route_destory;
+    
+    _sendpath->src_addr = sendpath->target_vdi->vdev->in_addr;
+    _sendpath->dst_addr = in_addr;
+    _sendpath->target_addr = target_in_addr;
+    ether_addr_copy(&__target_vdev->ether_addr, &_sendpath->src_eth_addr);
 
     /* 查ARP表 */
-    int ret = neigh_query(dest_device->instance->neigh, target_in_addr,
-        &sendpath->dst_eth_addr, &sendpath->target_vdi);
+    int ret = neigh_query(target_vdev->instance->neigh, target_in_addr,
+        &_sendpath->dst_eth_addr, &sendpath->target_vdi);
 
     /* 没有查询成功，保存起来下一次再查 */
     if (ret < 0)
     {
-        neigh_create_or_update(dest_device->instance->neigh,
-            target_in_addr, NULL, dest_device->vdi, 0);
-
-        sendpath->can_use = 0;
-        sendpath->target_addr = target_in_addr;
+        neigh_create_or_update(target_vdev->instance->neigh, target_in_addr, 
+            NULL, target_vdev->vdi, 0); sendpath->can_use = 0;
     }
     else
     {
         sendpath->can_use = 1;
     }
 
+    MR_DEBUG("Sendpath created: type=%d, in_addr=%u, target_in_addr=%u", type,
+        in_addr.s_addr, target_in_addr.s_addr);
+
     return sendpath;
 }
 
-//TODO: 路由表，默认网关实现
-struct mr_sendpath * marsio_sendpath_create_by_route(struct mr_instance * instance, struct in_addr addr)
+/* ======================================================================================== */
+/* 
+    \brief 创建SendPath的统一入口函数。
+
+    采用可变参数函数。根据创建类型的不同，传输参数的类型和数量也不同。
+
+    (_1_) type = MR_SENDPATH_DEV
+    struct mr_sendpath * marsio_sendpath_create(struct mr_instance * instance, int type = MR_SENDPATH_VDEV,
+                                                struct mr_vdev * vdev);
+    (_2_) type = MR_SENDPATH_ROUTE_NORMAL
+    struct mr_sendpath * marsio_sendpath_create(struct mr_instance * instance, int type = MR_SENDPATH_ROUTE_NORMAL,
+                                                struct in_addr in_addr);
+    (_3_) type = MR_SENDPATH_ROUTE_SPEC_DEV
+    struct mr_sendpath * marsio_sendpath_create(struct mr_instance * instance, int type = MR_SENDPATH_ROUTE_SPEC_DEV,
+                                                struct mr_vdev * vdev, struct in_addr in_addr);
+
+    \return Sendpath object handler
+*/
+
+struct mr_sendpath * marsio_sendpath_create(struct mr_instance * instance, int type, ...)
 {
-    struct mr_vdev * target_vdev = NULL;
-    struct in_addr target_in_addr;
+    va_list ptr_arg;
+    struct mr_sendpath * sendpath = 0;
 
-    for (int i = 0; i < instance->nr_vdevs; i++)
+    va_start(ptr_arg, type);
+    switch (type)
     {
-        struct mr_vdev * vdev = &instance->vdevs[i];
-        struct vdev * __vdev = vdev->vdi->vdev;
+    case MR_SENDPATH_VDEV:
+        sendpath = sendpath_vdev_create(instance, type, ptr_arg);
+        break;
 
-        if (!is_same_subnet(addr, __vdev->in_addr, __vdev->in_mask))
-            continue;
+    case MR_SENDPATH_ROUTE_NORMAL:
+    case MR_SENDPATH_ROUTE_SPEC_DEV:
+        sendpath = sendpath_route_create(instance, type, ptr_arg);
+        break;
 
-        target_in_addr = addr;
-        target_vdev = vdev;
+    default:
+        MR_ERROR("Invalided sendpath type in creating sendpath, Failed. ");
+        sendpath = NULL;
     }
 
-    if (target_vdev == NULL) return NULL;
-    return marsio_sendpath_create_by_droute(target_vdev, target_in_addr);
+    va_end(ptr_arg);
+    return sendpath;
 }
 
+int marsio_sendpath_option_set(struct mr_instance * instance, struct mr_sendpath * sendpath, int opt, ...)
+{
+    va_list ptr_arg;
+    int ret = 0;
 
-struct mr_sendpath * marsio_sendpath_create_by_vdev(struct mr_vdev * dest_device)
+    va_start(ptr_arg, opt);
+    if (sendpath->fn_option_set) ret = sendpath->fn_option_set(instance, sendpath, opt, ptr_arg);
+    va_end(ptr_arg);
+    return ret;
+}
+
+int marsio_sendpath_option_get(struct mr_instance * instance, struct mr_sendpath * sendpath, int opt, ...)
 {
-    struct mr_sendpath * sendpath = malloc(sizeof(struct mr_sendpath));
-    MR_VERIFY_MALLOC(sendpath);
-    memset(sendpath, 0, sizeof(struct mr_sendpath));
+    va_list ptr_arg;
+    int ret = 0;
 
-    sendpath->target_vdi = dest_device->vdi;
-    sendpath->can_use = 1;
-    return sendpath;
+    va_start(ptr_arg, opt);
+    if (sendpath->fn_option_get) ret = sendpath->fn_option_get(instance, sendpath, opt, ptr_arg);
+    va_end(ptr_arg);
+    return ret;
 }
 
 void marsio_sendpath_destory(struct mr_sendpath * sendpath)
 {
-    free(sendpath);
+    if (sendpath->fn_destory != NULL) sendpath->fn_destory(sendpath);
+    assert(sendpath->fn_destory != NULL);
+}
+
+/*  以下函数用于保持兼容性 */
+struct mr_sendpath * marsio_sendpath_create_by_droute(struct mr_vdev * dest_device, struct in_addr addr)
+{
+    return marsio_sendpath_create(dest_device->instance, MR_SENDPATH_ROUTE_SPEC_DEV, dest_device, addr);
+}
+
+struct mr_sendpath * marsio_sendpath_create_by_route(struct mr_instance * instance, struct in_addr addr)
+{
+    return marsio_sendpath_create(instance, MR_SENDPATH_ROUTE_NORMAL, addr);
+}
+
+struct mr_sendpath * marsio_sendpath_create_by_vdev(struct mr_vdev * dest_device)
+{
+    return marsio_sendpath_create(dest_device->instance, MR_SENDPATH_VDEV, dest_device);
 }
\ No newline at end of file