增加静态邻居表功能，修正send_burst_options函数可能存在的double-free的问题；

- 增加静态邻居表功能，可以从文件读入邻居定义；
- 原实现在外部传入NO_FREE选项且执行报文构建失败时，仍然将缓冲区释放，造成double-free。现修正。

3 files changed, 294 insertions, 18 deletions

diff --git a/app/include/mrapp.h b/app/include/mrapp.h
index f070761..c95e3cb 100644
--- a/app/include/mrapp.h
+++ b/app/include/mrapp.h
@@ -55,6 +55,19 @@ struct mr_thread_info
     cpu_id_t cpu_id;
 };
 
+/* 静态邻居表项 */
+struct mr_static_neigh_entry
+{
+    TAILQ_ENTRY(mr_static_entry) next;
+    struct in_addr in_addr;
+    struct ether_addr ether_addr;
+    char devsym[MR_SYMBOL_MAX];
+    char str_in_addr[MR_SYMBOL_MAX];
+    char str_ether_addr[MR_SYMBOL_MAX];
+};
+
+TAILQ_HEAD(mr_static_neigh_list, mr_static_neigh_entry);
+
 struct mr_instance
 {
     /* ==== 运行句柄 ==== */
@@ -83,6 +96,8 @@ struct mr_instance
     struct neighbour_manager * neigh;
 	/* 负载均衡器 */
 	struct distributer * dist_object;
+    /* 静态邻居表 */
+    struct mr_static_neigh_list static_neigh_list;
     /* 统计 */
     struct mrapp_stat stat[MR_SID_MAX];
 
diff --git a/app/src/marsio.c b/app/src/marsio.c
index 16fa589..a72eee8 100644
--- a/app/src/marsio.c
+++ b/app/src/marsio.c
@@ -5,6 +5,8 @@
 #include <rte_log.h>
 #include <rte_malloc.h>
 #include <rte_version.h>
+#include <rte_string_fns.h>
+
 #include <mrapp.h>
 #include <marsio.h>
 #include <arpa/inet.h>
@@ -28,7 +30,20 @@ struct mr_instance * _current_instance = NULL;
 
 #define MRAPP_MONIT_FILE_PATH               "/var/run/mrzcpd/mrmonit.app.%s"
 #define MRAPP_SERVICE_MONIT_FILE_PATH       "/var/run/mrzcpd/mrmonit.daemon"
-#define MRAPP_GLOBAL_CONF_FILE_PATH         "/etc/mrglobal.conf"
+#define MRAPP_GLOBAL_CONF_FILE_PATH         "/opt/mrzcpd/etc/mrglobal.conf"
+#define MRAPP_STATIC_NEIGH_FILE_PATH        "/opt/mrzcpd/etc/mrneigh.table"
+
+#ifndef MRAPP_DEFAULT_NEIGH_TABLE_MAX_ENTRIES
+#define MRAPP_DEFAULT_NEIGH_TABLE_MAX_ENTRIES           4096
+#endif
+
+#ifndef MRAPP_DEFAULT_NEIGH_TABLE_TIMEOUT
+#define MRAPP_DEFAULT_NEIGH_TABLE_TIMEOUT               0
+#endif
+
+#ifndef MRAPP_DEFAULT_NEIGH_ARP_SEND_INTERVAL
+#define MRAPP_DEFAULT_NEIGH_ARP_SEND_INTERVAL           1
+#endif
 
 /* 写入Command参数 */
 static void __write_arg(char * eal_argv[], unsigned int * eal_argc,
@@ -113,14 +128,14 @@ static void mrapp_eal_init(struct mr_instance * instance)
     cpu_set_t __cpu_set;
     ret = pthread_getaffinity_np(pthread_self(), sizeof(cpu_set_t), &__cpu_set);
     MR_VERIFY_2(ret >= 0, "Cannot get init thread affinity: %s", strerror(errno));
-        
+
     rte_openlog_stream(stderr);
     ret = rte_eal_init(eal_argc, eal_argv);
     MR_VERIFY_2(ret >= 0, "Cannot init EAL Enviorment, Failed");
 
     ret = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &__cpu_set);
     MR_VERIFY_2(ret >= 0, "Cannot set init thread affinity: %s", strerror(errno));
-    
+
     /* EAL环境初始化成功 */
     g_eal_inited = 1;
     return;
@@ -150,7 +165,7 @@ static int mrapp_ctrlmsg_init(struct mr_instance * instance)
     sockaddr_in.sin_family = AF_INET;
 
     /* 创建消息处理框架句柄 */
-    instance->ctrlmsg_handler = ctrlmsg_handler_create(CTRLMSG_HANDLER_MODE_CLIENT, sockaddr_in, NULL);
+    instance->ctrlmsg_handler = ctrlmsg_handler_create(CTRLMSG_HANDLER_MODE_CLIENT, sockaddr_in, NULL, -1);
     if (instance->ctrlmsg_handler == NULL) return RT_ERR;
     return RT_SUCCESS;
 }
@@ -232,6 +247,253 @@ static int mrapp_bpf_dumper_init(struct mr_instance * instance, struct mr_vdev *
     return RT_SUCCESS;
 }
 
+
+static unsigned __table_strip(char *str, unsigned len)

+{

+    int newlen = len;

+    if (len == 0)

+        return 0;

+

+    if (isspace(str[len - 1]))
+    {

+        /* strip trailing whitespace */

+        while (newlen > 0 && isspace(str[newlen - 1]))

+            str[--newlen] = '\0';

+    }

+

+    if (isspace(str[0]))
+    {

+        /* strip leading whitespace */

+        int i, start = 1;

+        while (isspace(str[start]) && start < newlen)

+            start++

+            ; /* do nothing */

+        newlen -= start;

+        for (i = 0; i < newlen; i++)

+            str[i] = str[i + start];

+        str[i] = '\0';

+    }
+

+    return newlen;

+}

+
+static const char * __table_readline(FILE * fp, char * buffer,
+    size_t sz_buffer, size_t * buffer_len, unsigned int * line_no)
+{
+    while (fgets(buffer, sz_buffer, fp) != NULL)
+    {
+        char *pos = NULL;

+        size_t len = strnlen(buffer, sz_buffer);
+        (*line_no)++;
+

+        if ((len >= sizeof(buffer) - 1) && (buffer[len - 1] != '\n'))
+        {
+            continue;

+        }
+

+        pos = memchr(buffer, '#', sz_buffer);

+        if (pos != NULL)
+        {

+            *pos = '\0';

+            len = pos - buffer;

+        }

+

+        len = __table_strip(buffer, len);
+        if (len == 0)
+        {
+            continue;
+        }
+
+        return buffer;
+    }
+
+    return NULL;
+}
+
+static int __table_split_line(const char * buffer, char str_tokens[MR_TOKENS_MAX][MR_STRING_MAX])
+{
+    char * __buffer = strdup(buffer);
+    assert(__buffer != NULL);
+    
+    char * token_ptr;
+    unsigned int total_nr_tokens = 0;
+
+    while ((token_ptr = strsep(&__buffer, " \t")) != NULL)
+    {
+        if (strlen(token_ptr) == 0) continue;
+        if (total_nr_tokens == MR_TOKENS_MAX) break;
+
+        strncpy(str_tokens[total_nr_tokens], token_ptr, MR_STRING_MAX);
+
+        int len = strnlen(str_tokens[total_nr_tokens], MR_STRING_MAX);
+        __table_strip(str_tokens[total_nr_tokens], len);
+
+        total_nr_tokens++;
+    }
+
+    free(__buffer);
+    return total_nr_tokens;
+}
+
+/* 邻居子系统管理器 */
+static int mrapp_neigh_init(struct mr_instance * instance)
+{
+    instance->neigh = malloc(sizeof(struct neighbour_manager));
+    MR_VERIFY_MALLOC(instance->neigh);
+
+    unsigned int neigh_max_entries;
+    unsigned int neigh_timeout;
+    unsigned int arp_send_interval;
+
+    /* 临时静态邻居表初始化 
+       该表仅限初始化时使用，设备打开后，该表项读取邻居子系统。
+    */
+    TAILQ_INIT(&instance->static_neigh_list);
+
+    MESA_load_profile_uint_def(instance->g_cfgfile_path, "neigh", "neigh_table_max_entries",
+        &neigh_max_entries, MRAPP_DEFAULT_NEIGH_TABLE_MAX_ENTRIES);
+    MESA_load_profile_uint_def(instance->g_cfgfile_path, "neigh", "neigh_table_timeout",
+        &neigh_timeout, MRAPP_DEFAULT_NEIGH_TABLE_TIMEOUT);
+    MESA_load_profile_uint_def(instance->g_cfgfile_path, "neigh", "arp_send_interval",
+        &arp_send_interval, MRAPP_DEFAULT_NEIGH_ARP_SEND_INTERVAL);
+
+    /* 邻居子系统初始化 */
+    if (neighbour_mamanger_init(instance->neigh, instance->appsym,
+        neigh_max_entries, neigh_timeout, arp_send_interval) != RT_SUCCESS)
+    {
+        goto err;
+    }
+
+    /* 读静态配置表 */
+    FILE * fp_static_neigh_table = fopen(MRAPP_STATIC_NEIGH_FILE_PATH, "r");
+    if (fp_static_neigh_table == NULL)
+    {
+        MR_DEBUG("Skip reading static neighbour table from file %s.", 
+            MRAPP_STATIC_NEIGH_FILE_PATH); goto success;
+    }
+
+    /* 按行读 */
+    char __tb_line_buffer[MR_STRING_MAX];
+    size_t __tb_buffer_len = 0;
+    unsigned int __tb_line_no = 0;
+
+    while (__table_readline(fp_static_neigh_table,
+        __tb_line_buffer, sizeof(__tb_line_buffer), &__tb_buffer_len, &__tb_line_no) != NULL)
+    {
+        /* 拆分，第一列为IP地址，第二列为MAC地址 */
+        char __split_str[MR_TOKENS_MAX][MR_STRING_MAX];
+        memset(__split_str, 0, sizeof(__split_str));
+
+        int nr_tokens = __table_split_line(__tb_line_buffer, __split_str);
+        if (nr_tokens != 3)
+        {
+            MR_WARNING("Table: %s, line: %u: Invalid line format, must have 3 column, ignore.",
+                MRAPP_STATIC_NEIGH_FILE_PATH, __tb_line_no); continue;
+        }
+        
+        /* 读IP地址、MAC地址 */
+        const char * str_in_addr = __split_str[0];
+        const char * str_ether_addr = __split_str[1];
+        const char * str_device = __split_str[2];
+        
+        /* 字符串转换 */
+        struct in_addr in_addr;
+        struct ether_addr ether_addr;
+
+        int ret = inet_pton(AF_INET, str_in_addr, &in_addr);
+        if (ret < 0)
+        {
+            MR_WARNING("Table: %s, line: %u: Invaild IP address format, ignore. ",
+                MRAPP_STATIC_NEIGH_FILE_PATH, __tb_line_no); continue;
+        }
+
+        ret = sscanf(str_ether_addr, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx",
+            &ether_addr.addr_bytes[0], &ether_addr.addr_bytes[1],
+            &ether_addr.addr_bytes[2], &ether_addr.addr_bytes[3],
+            &ether_addr.addr_bytes[4], &ether_addr.addr_bytes[5]);
+
+        if (ret != 6)
+        {
+            MR_WARNING("Table: %s, line: %u: Invaild MAC address format, ignore. ",
+                MRAPP_STATIC_NEIGH_FILE_PATH, __tb_line_no); continue;
+        }
+
+        /* 检查重复的表项 */
+        struct mr_static_neigh_entry * __neigh_entry_iter;
+        unsigned int is_dup_entry = 0;
+
+        TAILQ_FOREACH(__neigh_entry_iter, &instance->static_neigh_list, next)
+        {
+            if (__neigh_entry_iter->in_addr.s_addr != in_addr.s_addr) continue;
+
+            /* 重复表项，告警 */
+            MR_WARNING("table: %s, line: %u: Duplicate entry, %s->%s, ignore.",
+                MRAPP_STATIC_NEIGH_FILE_PATH, __tb_line_no, str_in_addr, str_ether_addr);
+            is_dup_entry = 1;  break;
+        }
+
+        /* 重复表项跳过 */
+        if (is_dup_entry) continue;
+
+        /* 插入临时的静态邻居表 */
+        struct mr_static_neigh_entry * __neigh_entry = malloc(sizeof(struct mr_static_neigh_entry));
+        memset(__neigh_entry, 0, sizeof(struct mr_static_neigh_entry));
+
+        __neigh_entry->in_addr = in_addr;
+        __neigh_entry->ether_addr = ether_addr;
+        strncpy(__neigh_entry->devsym, str_device, sizeof(__neigh_entry->devsym));
+
+        /* 拷贝IP地址、MAC地址的字符串形式，便于后面显示日志信息 */
+        strncpy(__neigh_entry->str_in_addr, str_in_addr, sizeof(__neigh_entry->str_in_addr));
+        strncpy(__neigh_entry->str_ether_addr, str_ether_addr, sizeof(__neigh_entry->str_ether_addr));
+        
+        TAILQ_INSERT_TAIL(&instance->static_neigh_list, __neigh_entry, next);
+    }
+    
+success:
+    return RT_SUCCESS;
+
+err:
+    if (instance->neigh != NULL)
+    {
+        free(instance->neigh);
+        instance->neigh = NULL;
+    }
+
+    return RT_ERR;
+}
+
+/* 邻居子系统，延迟初始化
+   设备初始化时初始化该部分
+*/
+static int mrapp_neigh_device_init(struct mr_instance * instance, struct mr_vdev * vdev)
+{
+    struct mr_static_neigh_entry * __neigh_entry_iter;
+    TAILQ_FOREACH(__neigh_entry_iter, &instance->static_neigh_list, next)
+    {
+        if (strncmp(vdev->devsym, __neigh_entry_iter->devsym, sizeof(vdev->devsym)) != 0)
+            continue;
+
+        int ret = neigh_create_or_update(instance->neigh, __neigh_entry_iter->in_addr,
+            &__neigh_entry_iter->ether_addr, vdev->vdi, 1);
+
+        if (ret < 0)
+        {
+            MR_WARNING("Static neighbour %s->%s on device %s failed, Ignore.",
+                __neigh_entry_iter->str_in_addr, 
+                __neigh_entry_iter->str_ether_addr, 
+                __neigh_entry_iter->devsym);
+
+            continue;
+        }
+
+        MR_INFO("Static neighbour: %s->%s, device: %s.", __neigh_entry_iter->str_in_addr,
+            __neigh_entry_iter->str_ether_addr, __neigh_entry_iter->devsym);
+    }
+
+    return RT_SUCCESS;
+}
+
 /* 注册应用 */
 static int mrapp_app_register(struct mr_instance * instance)
 {
@@ -446,6 +708,8 @@ struct mr_vdev * marsio_open_device(struct mr_instance * instance,
 
     /* 调试捕包 */
     mrapp_bpf_dumper_init(instance, vdev);
+    /* 静态邻居表 */
+    mrapp_neigh_device_init(instance, vdev);
     return vdev;
 }
 
@@ -576,13 +840,10 @@ int marsio_init(struct mr_instance * instance, const char * appsym)
         goto err;
     }
 
-    instance->neigh = malloc(sizeof(struct neighbour_manager));
-    MR_VERIFY_MALLOC(instance->neigh);
-
-    if (neighbour_mamanger_init(instance->neigh,
-        instance->appsym, 2048, 0, 1) != RT_SUCCESS)
+    /* 邻居子系统 */
+    if (mrapp_neigh_init(instance) != RT_SUCCESS)
     {
-        MR_ERROR("Neighbour management initialization failed.");
+        MR_ERROR("Neighbour initialization failed.");
         goto err;
     }
 
diff --git a/app/src/rawio.c b/app/src/rawio.c
index 51df2bd..1423bc3 100644
--- a/app/src/rawio.c
+++ b/app/src/rawio.c
@@ -172,14 +172,6 @@ int marsio_send_burst_with_options(struct mr_sendpath * sendpath, queue_id_t sid
         __rte_mbuf_sanity_check(mbufs[i], i);
     }
 
-    if (__send_burst_packet_construct(sendpath, sid, mbufs, nr_mbufs) < 0)
-        goto err;
-
-    if (options & MARSIO_SEND_OPT_REHASH)
-    {
-        distributer_caculate(sendpath->instance->dist_object, __mbufs, nr_mbufs);
-    }
-
     if (options & MARSIO_SEND_OPT_NO_FREE)
     {
         for (int i = 0; i < nr_mbufs; i++) rte_pktmbuf_refcnt_update(__mbufs[i], 1);
@@ -190,6 +182,14 @@ int marsio_send_burst_with_options(struct mr_sendpath * sendpath, queue_id_t sid
         for (int i = 0; i < nr_mbufs; i++) __mbufs[i]->ol_flags |= CTRL_MBUF_FLAG;
     }
 
+    if (__send_burst_packet_construct(sendpath, sid, mbufs, nr_mbufs) < 0)
+        goto err;
+
+    if (options & MARSIO_SEND_OPT_REHASH)
+    {
+        distributer_caculate(sendpath->instance->dist_object, __mbufs, nr_mbufs);
+    }
+
     hash_t hash[MR_BURST_MAX];
     for (int i = 0; i < nr_mbufs; i++) hash[i] = __mbufs[i]->hash.usr;