✨ feat(TSG-11533): LB节点支持动态添加规则

LB节点支持动态添加规则并支持RCU框架

7 files changed, 873 insertions, 59 deletions

diff --git a/service/src/node_lb.c b/service/src/node_lb.c
index 58b121f..58817d5 100644
--- a/service/src/node_lb.c
+++ b/service/src/node_lb.c
@@ -8,6 +8,7 @@
 #include <rte_graph.h>
 #include <rte_graph_worker.h>
 #include <cJSON.h>
+#include <rte_rcu_qsbr.h>
 
 #ifndef MR_LB_MAX_GROUP
 #define MR_LB_MAX_GROUP  1024
@@ -41,6 +42,15 @@
 #define MR_LB_DEV_VALID  2
 #endif
 
+/* Dynamic Load Balance Topic */
+#ifndef MR_LB_TOPIC_SINGLE_RULE_ADD
+#define MR_LB_TOPIC_SINGLE_RULE_ADD                             "LBSingleRuleAdd"
+#endif
+
+#ifndef MR_LB_TOPIC_SINGLE_RULE_DEL
+#define MR_LB_TOPIC_SINGLE_RULE_DEL                             "LBSingleRuleDel"
+#endif
+
 /* Dynamic LB Rule Field */
 #ifndef MR_LB_CJSON_KEY_GROUP_ID
 #define MR_LB_CJSON_KEY_GROUP_ID                                "Id"
@@ -66,6 +76,23 @@
 #define MR_LB_CJSON_KEY_DEV_TYPE                                "DevType"
 #endif
 
+
+#ifndef MR_LB_RPC_RESULT
+#define MR_LB_RPC_RESULT                                        "Result"
+#endif
+
+#ifndef MR_LB_RPC_RESULT_ERR_INFO
+#define MR_LB_RPC_RESULT_ERR_INFO                               "ErrInfo"
+#endif
+
+#ifndef MR_LB_RPC_RESULT_ERR_DEV_ID
+#define MR_LB_RPC_RESULT_ERR_DEV_ID                             "DevId"
+#endif
+
+#ifndef MR_LB_RPC_RESULT_ERR_DEV_ERR_CODE
+#define MR_LB_RPC_RESULT_ERR_DEV_ERR_CODE                       "ErrCode"
+#endif
+
 #define MR_LB_STAT_ADD(st, gid, counter, value)                                                                 \
 do                                                                                                                    \
 {                                                                                                                     \
@@ -82,9 +109,10 @@ enum {
 /* Dev Type */
 enum
 {
-    NORMAL_DEV = 0,
-    ACTIVE_DEV,
-    BACKUP_DEV,
+    DEV_TYPE_NORMAL = 0,
+    DEV_TYPE_ACTIVE,
+    DEV_TYPE_BACKUP,
+    DEV_TYPE_MAX,
 };
 
 /* Dev Flag */
@@ -97,15 +125,35 @@ enum
 /* Group Mode */
 enum
 {
-    ACTIVE_BACKUP_MODE = 0,
-    BALANCE_MODE
+    GROUP_MODE_ACTIVE_BACKUP = 0,
+    GROUP_MODE_BALANCE,
+    GROUP_MODE_MAX
 };
 
 /* Group State */
 enum
 {
-    GROUP_STATE_UN_USE = 0,
-    GROUP_STATE_IN_USE
+    GROUP_STATUS_UN_USE = 0,
+    GROUP_STAUS_IN_USE
+};
+
+/* Dynamic Load Balance Rule Add Deal Result */
+enum {
+    DYNAMIC_LB_RULE_ADD_SUCESS = 0,
+    DYNAMIC_LB_RULE_ADD_NUM_OUT_OF_MAX,
+    DYNAMIC_LB_RULE_ADD_NO_FREE_GROUP_ITEM,
+    DYNAMIC_LB_RULE_ADD_GROUP_ID_NO_SET,
+    DYNAMIC_LB_RULE_ADD_GROUP_MODE_NO_SET,
+    DYNAMIC_LB_RULE_ADD_DEV_NUM_NO_SET,
+    DYNAMIC_LB_RULE_ADD_GROUP_MODE_INVALID,
+    DYNAMIC_LB_RULE_ADD_DEV_NUM_INVALID,
+    DYNAMIC_LB_RULE_ADD_DEV_ITEM_NO_SET,
+    DYNAMIC_LB_RULE_ADD_DEV_NAME_NO_SET,
+    DYNAMIC_LB_RULE_ADD_DEV_NO_FIND_TX_NODE,
+    DYNAMIC_LB_RULE_ADD_DEV_NO_FIND_THE_DEV,
+    DYNAMIC_LB_RULE_ADD_DEV_TYPE_NO_SET,
+    DYNAMIC_LB_RULE_ADD_DEV_TYPE_INVALID,
+    DYNAMIC_LB_RULE_ADD_REPEATED_RULE
 };
 
 /* LB Dev Struct */
@@ -121,10 +169,9 @@ struct lb_vdev
 /* LB Group Struct */
 struct lb_group
 {
-    uint8_t group_state;
+    uint8_t group_status;
     uint8_t group_mode;
     uint16_t group_id;
-    uint16_t master_vdev_id;
     uint16_t vdev_num;
     struct lb_vdev vdev_buf[MR_LB_MAX_DEV_FOR_SINGLE_GROUP];
 };
@@ -148,8 +195,9 @@ struct lb_stat_per_lcore
 /* LB Main Struct */
 struct node_lb_main
 {
-    struct lb_management * lb_manage;
-    struct lb_stat_per_lcore stat_per_graph[RTE_MAX_LCORE];
+    struct rte_rcu_qsbr * qsv;                                          /* Quiescent State variable */
+    struct lb_management * lb_manage;                                   /* Load Banlance Management */
+    struct lb_stat_per_lcore stat_per_graph[RTE_MAX_LCORE];             /* Load Banlance Stat */
 };
 
 /* Globle LB Main */
@@ -171,12 +219,28 @@ struct lb_management * create_lb_management(void)
     return lb_manage;
 }
 
+/* Free A Load Balance Management Item */
+void free_lb_management(struct lb_management * lb_manage)
+{
+    FREE(lb_manage);
+}
+
+/* LB Management Copy */
+void lb_management_copy(struct lb_management * src,struct lb_management * dst)
+{
+    dst->group_num = src->group_num;
+    memcpy(dst->lb_groups,src->lb_groups,sizeof(dst->lb_groups));
+}
+
 /* Get Free Group Item Id */
 int get_free_group_item_id(struct lb_management * lb_manage,uint16_t * item_id)
 {
+    struct lb_group * group_item = NULL;
+
     for (uint16_t i = 0; i < MR_LB_MAX_GROUP; i++)
     {
-        if (globle_lb_main->lb_manage->lb_groups[i].group_state == GROUP_STATE_UN_USE)
+        group_item = &globle_lb_main->lb_manage->lb_groups[i];
+        if (group_item->group_status == GROUP_STATUS_UN_USE)
         {
             *item_id = i;
             return RT_SUCCESS;
@@ -188,11 +252,14 @@ int get_free_group_item_id(struct lb_management * lb_manage,uint16_t * item_id)
 /* Check Group Id */
 int lb_check_group_id(uint16_t group_id)
 {
+    struct lb_group * group_item = NULL;
+
     for (int i = 0; i < MR_LB_MAX_GROUP; i++)
     {
-        if (globle_lb_main->lb_manage->lb_groups[i].group_state == GROUP_STATE_IN_USE)
+        group_item = &globle_lb_main->lb_manage->lb_groups[i];
+        if (group_item->group_status == GROUP_STAUS_IN_USE)
         {
-            if (globle_lb_main->lb_manage->lb_groups[i].group_id == group_id)
+            if (group_item->group_id == group_id)
             {
                 return RT_SUCCESS;
             }
@@ -201,6 +268,40 @@ int lb_check_group_id(uint16_t group_id)
     return RT_ERR;
 }
 
+/* Load Balance Repeated Rule Check */
+int lb_repeated_rule_check(struct lb_management * lb_manage,struct lb_group * check_group_item)
+{
+    struct lb_group * group_item = NULL;
+
+    for (int i = 0; i < MR_LB_MAX_GROUP; i++)
+    {
+        group_item = &lb_manage->lb_groups[i];
+        if (group_item->group_status == GROUP_STAUS_IN_USE)
+        {
+            if ((group_item->group_mode == check_group_item->group_mode) || (group_item->vdev_num == check_group_item->vdev_num))
+            {
+                uint16_t vdev_num = 0;
+                for (int j = 0; j < MR_LB_MAX_DEV_FOR_SINGLE_GROUP; j++)
+                {
+                    if (group_item->vdev_buf[j].vdev_flag == DEV_ENABLE)
+                    {
+                        if (group_item->vdev_buf[j].vdev == check_group_item->vdev_buf[j].vdev )
+                        {
+                            vdev_num ++;
+                        }
+                    }
+                }
+
+                if (check_group_item->vdev_num == vdev_num)
+                {
+                    return RT_ERR;
+                }
+            }
+        }
+    }
+    return RT_SUCCESS;
+}
+
 /************************************* LB Config **************************************/
 /* Parser The Local LB Config */
 int parser_local_lb_conf(struct sc_main * sc, struct lb_management * lb_manage)
@@ -245,7 +346,7 @@ int parser_local_lb_conf(struct sc_main * sc, struct lb_management * lb_manage)
         }
 
         lb_group_item = &lb_manage->lb_groups[item_id];
-        lb_group_item->group_state = GROUP_STATE_IN_USE;
+        lb_group_item->group_status = GROUP_STAUS_IN_USE;
         lb_group_item->group_id = i;
 
         snprintf(group_str_section, sizeof(group_str_section), "group:%d", i);
@@ -259,7 +360,7 @@ int parser_local_lb_conf(struct sc_main * sc, struct lb_management * lb_manage)
         /* Save Mode */
         if (strcmp(str_buf, "balance_mode") == 0)
         {
-            lb_group_item->group_mode = BALANCE_MODE;
+            lb_group_item->group_mode = GROUP_MODE_BALANCE;
         }
         else
         {
@@ -303,7 +404,7 @@ int parser_local_lb_conf(struct sc_main * sc, struct lb_management * lb_manage)
             /* Save Type */
             if (strcmp(str_buf, "normal") == 0)
             {
-                lb_group_item->vdev_buf[j].vdev_type = NORMAL_DEV;
+                lb_group_item->vdev_buf[j].vdev_type = DEV_TYPE_NORMAL;
             }
             else
             {
@@ -367,59 +468,296 @@ void dump_lb_config(struct lb_management * lb_manage)
     MR_INFO("LB Config:");
     MR_INFO("   Group Total Num                : %u",lb_manage->group_num);
 
-    for (int i = 0; i < lb_manage->group_num; i++)
+    for (int i = 0; i < MR_LB_MAX_GROUP; i++)
     {
         struct lb_group * lb_groups = &lb_manage->lb_groups[i];
 
-        MR_INFO(" ");
-        MR_INFO("   Group Id                       : %d",i);
-        switch (lb_groups->group_mode)
-        {
-            case BALANCE_MODE:
-                MR_INFO("     Group Mode                   : Balance Mode");
-                break;
-
-            default:
-                break;
-        }
-
-        MR_INFO("     Dev Number                  : %u",lb_groups->vdev_num);
-        for (int j = 0; j < lb_groups->vdev_num; j++)
+        if (lb_groups->group_status == GROUP_STAUS_IN_USE)
         {
-            struct lb_vdev * fv = &lb_groups->vdev_buf[j];
-
-            MR_INFO(" ");
-            MR_INFO("       Dev Name                  : %s",fv->vdevsym);
-
-            switch (fv->vdev_type)
+            MR_INFO("   Group Id                       : %u",lb_groups->group_id);
+            switch (lb_groups->group_mode)
             {
-                case NORMAL_DEV:
-                    MR_INFO("       Dev Type                  : Normal");
+                case GROUP_MODE_BALANCE:
+                    MR_INFO("     Group Mode                   : Balance Mode");
                     break;
+
                 default:
                     break;
             }
 
-            switch (fv->vdev->link_status)
+            MR_INFO("     Dev Number                   : %u",lb_groups->vdev_num);
+            for (int j = 0; j < MR_LB_MAX_DEV_FOR_SINGLE_GROUP; j++)
             {
-            case LINK_UP:
-                MR_INFO("       Dev Link State            : Up");
-                break;
+                struct lb_vdev * vdev = &lb_groups->vdev_buf[j];
+                if (vdev->vdev_flag == DEV_ENABLE)
+                {
+                    MR_INFO("       Dev Name                   : %s",vdev->vdevsym);
+
+                    switch (vdev->vdev_type)
+                    {
+                        case DEV_TYPE_NORMAL:
+                            MR_INFO("        Dev Type                   : Normal");
+                            break;
+                        default:
+                            break;
+                    }
+
+                    switch (vdev->vdev->link_status)
+                    {
+                    case LINK_UP:
+                        MR_INFO("        Dev Link State             : Up");
+                        break;
+
+                    default:
+                        MR_INFO("        Dev Link State             : Down");
+                        break;
+                    }
+
+                    MR_INFO("        Tx Node Index              : %u",vdev->tx_node_index);
+                }
+            }
+        }
+    }
+}
+
+/************************************** LB Dynamic Rull Process **************************************/
+/* Parse LB Rules For Add */
+int parse_lb_rule_for_add(struct sc_main * sc,struct lb_management * lb_manage,cJSON * j_rule,cJSON *response)
+{
+    uint8_t group_mode = 0;
+    uint16_t item_id = 0,group_id = 0,vdev_num = 0;
+    struct lb_group lb_group_item = {};
+    struct lb_group * group_item;
+    char dev_str_section[MR_STRING_MAX];
+    cJSON * j_group_id = NULL,* j_group_mode = NULL,* j_dev_num = NULL,* j_dev = NULL,* j_dev_name = NULL,* j_dev_type = NULL;
+
+
+    /* Get Free Group Item Id */
+    if (get_free_group_item_id(lb_manage,&item_id) == RT_ERR)
+    {
+        return DYNAMIC_LB_RULE_ADD_NO_FREE_GROUP_ITEM;
+    }
+
+    group_item = &lb_group_item;
 
-            default:
-                MR_INFO("       Dev Link State            : Down");
+    /* Group Id */
+    j_group_id = cJSON_GetObjectItem(j_rule,MR_LB_CJSON_KEY_GROUP_ID);
+    /* Group Mode */
+    j_group_mode = cJSON_GetObjectItem(j_rule,MR_LB_CJSON_KEY_GROUP_MODE);
+    /* Dev Num */
+    j_dev_num = cJSON_GetObjectItem(j_rule,MR_LB_CJSON_KEY_DEV_NUM);
+
+    if (j_group_id == NULL)
+    {
+        return DYNAMIC_LB_RULE_ADD_GROUP_ID_NO_SET;
+    }
+
+    if (j_group_mode == NULL)
+    {
+        return DYNAMIC_LB_RULE_ADD_GROUP_MODE_NO_SET;
+    }
+
+    if (j_dev_num == NULL)
+    {
+        return DYNAMIC_LB_RULE_ADD_DEV_NUM_NO_SET;
+    }
+
+    if ((uint8_t)j_group_mode->valuedouble >= GROUP_MODE_MAX)
+    {
+        return DYNAMIC_LB_RULE_ADD_GROUP_MODE_INVALID;
+    }
+
+    if (((uint16_t)j_dev_num->valuedouble == MR_LB_INVALID_RULE_ARG) || ((uint16_t)j_dev_num->valuedouble > MR_LB_MAX_DEV_FOR_SINGLE_GROUP))
+    {
+        return DYNAMIC_LB_RULE_ADD_DEV_NUM_INVALID;
+    }
+
+    group_id = (uint16_t)j_group_id->valuedouble;
+    group_mode = (uint8_t)j_group_mode->valuedouble;
+    vdev_num = (uint16_t)j_dev_num->valuedouble;
+
+    /* Parsing All Device */
+    for (int i = 0; i < vdev_num; i++)
+    {
+        int vdev_flg = MR_LB_DEV_INVALID;
+        uint32_t count;
+        char **next_edges;
+        char tx_node_name[MR_SYMBOL_MAX];
+        struct lb_vdev * vdev_item = NULL;
+        struct vdev * vdev = NULL;
+        vdev_item = &group_item->vdev_buf[i];
+
+        snprintf(dev_str_section, sizeof(dev_str_section), "%s%d", MR_LB_CJSON_KEY_DEV, i);
+        j_dev = cJSON_GetObjectItem(j_rule,dev_str_section);
+
+        if (j_dev == NULL)
+        {
+            cJSON *err_info = cJSON_CreateObject();
+            cJSON_AddNumberToObject(err_info, MR_LB_CJSON_KEY_GROUP_ID, group_id);
+            cJSON_AddNumberToObject(err_info, MR_LB_RPC_RESULT_ERR_DEV_ID, i);
+            cJSON_AddNumberToObject(err_info, MR_LB_RPC_RESULT_ERR_DEV_ERR_CODE, DYNAMIC_LB_RULE_ADD_DEV_ITEM_NO_SET);
+            cJSON_AddItemToObject(response,MR_LB_RPC_RESULT_ERR_INFO,err_info);
+            return DYNAMIC_LB_RULE_ADD_DEV_ITEM_NO_SET;
+        }
+
+        j_dev_name = cJSON_GetObjectItem(j_dev,MR_LB_CJSON_KEY_DEV_NAME);
+        if (j_dev_name == NULL)
+        {
+            cJSON *err_info = cJSON_CreateObject();
+            cJSON_AddNumberToObject(err_info, MR_LB_CJSON_KEY_GROUP_ID, group_id);
+            cJSON_AddNumberToObject(err_info, MR_LB_RPC_RESULT_ERR_DEV_ID, i);
+            cJSON_AddNumberToObject(err_info, MR_LB_RPC_RESULT_ERR_DEV_ERR_CODE, DYNAMIC_LB_RULE_ADD_DEV_NAME_NO_SET);
+            cJSON_AddItemToObject(response,MR_LB_RPC_RESULT_ERR_INFO,err_info);
+            return DYNAMIC_LB_RULE_ADD_DEV_NAME_NO_SET;
+        }
+
+        /* Check The Dev Name */
+        memcpy(vdev_item->vdevsym,j_dev_name->valuestring,sizeof(vdev_item->vdevsym));
+        snprintf(tx_node_name, sizeof(tx_node_name), "shmdev_tx-%s",vdev_item->vdevsym);
+        rte_node_t node_id = rte_node_from_name("lb");
+        count = rte_node_edge_get(node_id, NULL);
+        next_edges = malloc(count);
+        count = rte_node_edge_get(node_id, next_edges);
+
+        for (int ni = 0; ni < count; ni++)
+        {
+            if (strncmp(next_edges[ni],tx_node_name,sizeof(tx_node_name)) == 0)
+            {
+                vdev_item->tx_node_index = ni;
+                vdev_flg = MR_LB_DEV_NEXT_NODE_VALID;
                 break;
             }
+        }
+
+        free(next_edges);
+
+        if (vdev_flg != MR_LB_DEV_NEXT_NODE_VALID)
+        {
+            cJSON *err_info = cJSON_CreateObject();
+            cJSON_AddNumberToObject(err_info, MR_LB_CJSON_KEY_GROUP_ID, group_id);
+            cJSON_AddNumberToObject(err_info, MR_LB_RPC_RESULT_ERR_DEV_ID, i);
+            cJSON_AddNumberToObject(err_info, MR_LB_RPC_RESULT_ERR_DEV_ERR_CODE, DYNAMIC_LB_RULE_ADD_DEV_NO_FIND_TX_NODE);
+            cJSON_AddItemToObject(response,MR_LB_RPC_RESULT_ERR_INFO,err_info);
+            return DYNAMIC_LB_RULE_ADD_DEV_NO_FIND_TX_NODE;
+        }
+
+        vdev = vdev_lookup(sc->vdev_main, vdev_item->vdevsym);
+        if (vdev == NULL)
+        {
+            cJSON *err_info = cJSON_CreateObject();
+            cJSON_AddNumberToObject(err_info, MR_LB_CJSON_KEY_GROUP_ID, group_id);
+            cJSON_AddNumberToObject(err_info, MR_LB_RPC_RESULT_ERR_DEV_ID, i);
+            cJSON_AddNumberToObject(err_info, MR_LB_RPC_RESULT_ERR_DEV_ERR_CODE, DYNAMIC_LB_RULE_ADD_DEV_NO_FIND_THE_DEV);
+            cJSON_AddItemToObject(response,MR_LB_RPC_RESULT_ERR_INFO,err_info);
+            return DYNAMIC_LB_RULE_ADD_DEV_NO_FIND_THE_DEV;
+        }
+        /* Save The vdev */
+        vdev_item->vdev = vdev;
 
-            MR_INFO("       Tx Node Index              : %u",fv->tx_node_index);
+        /* Parsing Dev Type */
+        j_dev_type = cJSON_GetObjectItem(j_dev,MR_LB_CJSON_KEY_DEV_TYPE);
+        if (j_dev_type == NULL)
+        {
+            cJSON *err_info = cJSON_CreateObject();
+            cJSON_AddNumberToObject(err_info, MR_LB_CJSON_KEY_GROUP_ID, group_id);
+            cJSON_AddNumberToObject(err_info, MR_LB_RPC_RESULT_ERR_DEV_ID, i);
+            cJSON_AddNumberToObject(err_info, MR_LB_RPC_RESULT_ERR_DEV_ERR_CODE, DYNAMIC_LB_RULE_ADD_DEV_TYPE_NO_SET);
+            cJSON_AddItemToObject(response,MR_LB_RPC_RESULT_ERR_INFO,err_info);
+            return DYNAMIC_LB_RULE_ADD_DEV_TYPE_NO_SET;
         }
+
+        vdev_item->vdev_type = (uint8_t)j_dev_type->valuedouble;
+        if (vdev_item->vdev_type >= DEV_TYPE_MAX)
+        {
+            cJSON *err_info = cJSON_CreateObject();
+            cJSON_AddNumberToObject(err_info, MR_LB_CJSON_KEY_GROUP_ID, group_id);
+            cJSON_AddNumberToObject(err_info, MR_LB_RPC_RESULT_ERR_DEV_ID, i);
+            cJSON_AddNumberToObject(err_info, MR_LB_RPC_RESULT_ERR_DEV_ERR_CODE, DYNAMIC_LB_RULE_ADD_DEV_TYPE_INVALID);
+            cJSON_AddItemToObject(response,MR_LB_RPC_RESULT_ERR_INFO,err_info);
+            return DYNAMIC_LB_RULE_ADD_DEV_TYPE_INVALID;
+        }
+
+        vdev_item->vdev_flag = DEV_ENABLE;
     }
+
+    group_item->group_mode = group_mode;
+    group_item->group_id = group_id;
+    group_item->vdev_num = vdev_num;
+    group_item->group_status = GROUP_STAUS_IN_USE;
+
+    if (lb_repeated_rule_check(lb_manage,group_item) == RT_ERR)
+    {
+        return DYNAMIC_LB_RULE_ADD_REPEATED_RULE;
+    }
+
+    lb_manage->group_num ++;
+    memcpy(&lb_manage->lb_groups[item_id],group_item,sizeof(struct lb_group));
+
+    return DYNAMIC_LB_RULE_ADD_SUCESS;
+}
+
+/* Add A Single Dynamic Acl Rule Callback */
+int lb_single_rule_add(struct sc_main * sc,cJSON * j_rule,cJSON *response)
+{
+    int ret = RT_ERR;
+    struct lb_management * new_lb_management = NULL;
+    struct lb_management * old_lb_management = globle_lb_main->lb_manage;
+    struct rte_rcu_qsbr * qsv = globle_lb_main->qsv;
+
+    /* Create A New Acl Management */
+    new_lb_management = create_lb_management();
+
+    /* Check Current Rule Number */
+    if (old_lb_management->group_num >= MR_LB_MAX_GROUP )
+    {
+        ret = DYNAMIC_LB_RULE_ADD_NUM_OUT_OF_MAX;
+        goto add_err;
+    }
+
+    /* LB Management Copy  */
+    lb_management_copy(old_lb_management,new_lb_management);
+    ret = parse_lb_rule_for_add(sc,new_lb_management,j_rule,response);
+    if (ret == DYNAMIC_LB_RULE_ADD_SUCESS)
+    {
+        /* Update LB Management */
+        globle_lb_main->lb_manage = new_lb_management;
+
+        /* Wait All Thread Quiescent State */
+        rte_rcu_qsbr_synchronize(qsv, RTE_QSBR_THRID_INVALID);
+
+        /* Free Old Acl Management */
+        free_lb_management(old_lb_management);
+        dump_lb_config(new_lb_management);
+        goto add_success;
+    }
+
+add_err:
+    free_lb_management(new_lb_management);
+add_success:
+    return ret;
 }
 
+/* Single Dynamic LB Rule Add Request Handler */
+static int __lb_single_rule_add_request_handler(cJSON *req, cJSON **rsp, void *arg)
+{
+    int ret = 0;
+    cJSON *response = cJSON_CreateObject();
+    struct sc_main * sc = (struct sc_main *)arg;
+
+    ret = lb_single_rule_add(sc,req,response);
+
+    cJSON_AddNumberToObject(response, MR_LB_RPC_RESULT, ret);
+    *rsp = response;
+
+    return 0;
+}
+
+
 /* Init LB */
 int lb_init(struct sc_main * sc)
 {
     int ret = RT_SUCCESS;
+    size_t sz;
     struct lb_management * lb_manage;
     struct node_lb_main * lbm = ZMALLOC(sizeof(struct node_lb_main));
     MR_VERIFY_MALLOC(lbm);
@@ -429,6 +767,23 @@ int lb_init(struct sc_main * sc)
     globle_lb_main = lbm;
     sc->lb_node_main = lbm;
 
+    /* Register Load Balance Rule Add/Delete Request Handler */
+    if(rpc_server_recv_request_register(sc->rpc_srv_handler, MR_LB_TOPIC_SINGLE_RULE_ADD, __lb_single_rule_add_request_handler, sc) != 0)
+    {
+        MR_ERROR("The '%s' Request Register Failed !!! ",MR_LB_TOPIC_SINGLE_RULE_ADD);
+        return RT_ERR;
+    }
+
+    /* Init Load Balance Rcu */
+    sz = rte_rcu_qsbr_get_memsize(RTE_MAX_LCORE);
+    lbm->qsv = (struct rte_rcu_qsbr *)rte_zmalloc(NULL, sz, RTE_CACHE_LINE_SIZE);
+    rte_rcu_qsbr_init(lbm->qsv, RTE_MAX_LCORE);
+    for (uint32_t worker_id = 0; worker_id < sc->nr_io_thread; worker_id++)
+    {
+        uint32_t lcore_id = cpu_set_location(&sc->cpu_set_io, worker_id);
+        rte_rcu_qsbr_thread_register(lbm->qsv, lcore_id);
+    }
+
     /* 1. Parser The Local LB Config */
     ret = parser_local_lb_conf(sc,lb_manage);
 
@@ -441,7 +796,6 @@ int lb_init(struct sc_main * sc)
     return ret;
 }
 
-
 /************************************** LB Node **************************************/
 /* LB Node Init Function */
 static int lb_node_init(const struct rte_graph * graph, struct rte_node * node)
@@ -455,10 +809,17 @@ static __rte_always_inline uint16_t lb_node_process(struct rte_graph * graph, st
 {
     uint16_t n_left_from = 0, last_spec = 0;
     uint16_t next_index,next0;
+    uint32_t lcore_id = 0;
     rte_graph_t graph_id;
     struct rte_mbuf *mbuf0, ** pkts;
     void  **from;
     struct node_lb_main * lbm = NULL;
+    struct rte_rcu_qsbr * qsv;
+
+    /* Set Rcu Thread Online */
+    qsv = globle_lb_main->qsv;
+    lcore_id = rte_lcore_id();
+    rte_rcu_qsbr_thread_online(qsv, lcore_id);
 
     /* 1. Get Pkts Num And Pkts Buffer */
     n_left_from = cnt;
@@ -490,7 +851,7 @@ static __rte_always_inline uint16_t lb_node_process(struct rte_graph * graph, st
         {
             lb_group_item = &lbm->lb_manage->lb_groups[gid];
 
-            if (lb_group_item->group_mode == BALANCE_MODE)
+            if (lb_group_item->group_mode == GROUP_MODE_BALANCE)
             {
                 struct lb_vdev * fv = &lb_group_item->vdev_buf[0];
                 if (likely(fv->vdev->link_status == LINK_UP))
@@ -545,6 +906,9 @@ node_enqueue:
             rte_node_enqueue_x1(graph, node, next_index, from[i]);
     }
 
+    /* Update Quiescent State Counter */
+    rte_rcu_qsbr_quiescent(qsv, lcore_id);
+    rte_rcu_qsbr_thread_offline(qsv, lcore_id);
     return cnt;
 }
 
diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
index e67b622..e9c4551 100644
--- a/tools/CMakeLists.txt
+++ b/tools/CMakeLists.txt
@@ -6,4 +6,5 @@ add_subdirectory(dlogreader)
 add_subdirectory(devbind)
 add_subdirectory(systemd)
 add_subdirectory(mrctl)
-add_subdirectory(acl_rule_test)
\ No newline at end of file
+add_subdirectory(classifier_rule_test)
+add_subdirectory(lb_rule_test)
\ No newline at end of file
diff --git a/tools/acl_rule_test/CMakeLists.txt b/tools/acl_rule_test/CMakeLists.txt
deleted file mode 100644
index f909963..0000000
--- a/tools/acl_rule_test/CMakeLists.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-find_package(DPDK REQUIRED)
-find_package(SYSTEMD REQUIRED)
-
-include_directories(${CMAKE_SOURCE_DIR}/service/include)
-include_directories(${DPDK_INCLUDE_DIR})
-
-add_executable(acl_rule_test acl_rule_test.c)
-target_link_libraries(acl_rule_test MESA_prof_load_static infra m MESA_htable_static ${SYSTEMD_LIBRARIES})
-install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/acl_rule_test DESTINATION bin COMPONENT Program)
\ No newline at end of file
diff --git a/tools/classifier_rule_test/CMakeLists.txt b/tools/classifier_rule_test/CMakeLists.txt
new file mode 100644
index 0000000..4f41670
--- /dev/null
+++ b/tools/classifier_rule_test/CMakeLists.txt
@@ -0,0 +1,9 @@
+find_package(DPDK REQUIRED)
+find_package(SYSTEMD REQUIRED)
+
+include_directories(${CMAKE_SOURCE_DIR}/service/include)
+include_directories(${DPDK_INCLUDE_DIR})
+
+add_executable(classifier_rule_test classifier_rule_test.c)
+target_link_libraries(classifier_rule_test MESA_prof_load_static infra m MESA_htable_static ${SYSTEMD_LIBRARIES})
+install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/classifier_rule_test DESTINATION bin COMPONENT Program)
\ No newline at end of file
diff --git a/tools/acl_rule_test/acl_rule_test.c b/tools/classifier_rule_test/classifier_rule_test.c
index 6721693..6721693 100644
--- a/tools/acl_rule_test/acl_rule_test.c
+++ b/tools/classifier_rule_test/classifier_rule_test.c
diff --git a/tools/lb_rule_test/CMakeLists.txt b/tools/lb_rule_test/CMakeLists.txt
new file mode 100644
index 0000000..f24431f
--- /dev/null
+++ b/tools/lb_rule_test/CMakeLists.txt
@@ -0,0 +1,9 @@
+find_package(DPDK REQUIRED)
+find_package(SYSTEMD REQUIRED)
+
+include_directories(${CMAKE_SOURCE_DIR}/service/include)
+include_directories(${DPDK_INCLUDE_DIR})
+
+add_executable(lb_rule_test lb_rule_test.c)
+target_link_libraries(lb_rule_test MESA_prof_load_static infra m MESA_htable_static ${SYSTEMD_LIBRARIES})
+install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/lb_rule_test DESTINATION bin COMPONENT Program)
\ No newline at end of file
diff --git a/tools/lb_rule_test/lb_rule_test.c b/tools/lb_rule_test/lb_rule_test.c
new file mode 100644
index 0000000..b5f6004
--- /dev/null
+++ b/tools/lb_rule_test/lb_rule_test.c
@@ -0,0 +1,440 @@
+#include <unistd.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include<arpa/inet.h>
+#include <getopt.h>
+#include <stdlib.h>
+#include <ctrlmsg.h>
+#include <ctrlmsg_define.h>
+#include <cJSON.h>
+#include <MESA_prof_load.h>
+#include <rte_flow.h>
+#include <sc_flow.h>
+#include <rte_version.h>
+#include <rpc.h>
+
+#define GLOBAL_CONF_FILE_PATH                       "/opt/tsg/mrzcpd/etc/mrglobal.conf"
+#define MR_CLI_ARG_LIST                             "hcsdg:iep:"
+#define MR_CLI_RESPONSE_DISABLE                     0
+#define MR_CLI_RESPONSE_ENABLE                      1
+#define MR_CLI_CLOSE_DELAY                          100
+
+/* Dynamic Load Balance Topic */
+#ifndef MR_LB_TOPIC_SINGLE_RULE_ADD
+#define MR_LB_TOPIC_SINGLE_RULE_ADD                             "LBSingleRuleAdd"
+#endif
+
+#ifndef MR_LB_TOPIC_SINGLE_RULE_DEL
+#define MR_LB_TOPIC_SINGLE_RULE_DEL                             "LBSingleRuleDel"
+#endif
+
+#ifndef MR_LB_RPC_RESULT
+#define MR_LB_RPC_RESULT                                        "Result"
+#endif
+
+enum mr_cli_arg
+{
+    MR_CLI_DELETE_ALL = 1000,
+    MR_CLI_ADD_FOR_ID,
+    MR_CLI_DEL_FOR_ID,
+    MR_CLI_LOOP,
+    MR_CLI_REPEATED,
+};
+/* Instance */
+struct rpc_instance
+{
+    volatile unsigned  response_flag;
+    struct rpc_client_handler * handler;
+};
+
+/* The Global ariable */
+struct rpc_instance * rpc_instance = NULL;
+
+/* Command Line Options */
+static struct option cli_options[] =
+{
+    {"help", no_argument,NULL, 'h'},
+    {"add", required_argument,NULL, 'a'},
+    {"delete", required_argument,NULL, 'd'},
+    {"delete_all", no_argument,NULL, MR_CLI_DELETE_ALL},
+    {"add_id", required_argument,NULL, MR_CLI_ADD_FOR_ID},
+    {"delete_id", required_argument,NULL, MR_CLI_DEL_FOR_ID},
+    {"loop", no_argument,NULL, MR_CLI_LOOP},
+    {"repeated", no_argument,NULL, MR_CLI_REPEATED},
+};
+
+/* Rpc Create */
+struct rpc_instance * rpc_create()
+{
+    struct rpc_instance * instance;
+    instance = malloc(sizeof(struct rpc_instance));
+    memset(instance, 0, sizeof(struct rpc_instance));
+
+    instance->handler = rpc_client_alloc_handler(RPC_CLIENT_MODE_ASYNC);
+    if(instance->handler == NULL)
+    {
+        printf("Alloc client handler failed\n");
+        return NULL;
+    }
+
+    instance->response_flag = MR_CLI_RESPONSE_DISABLE;
+
+    return instance;
+}
+
+void help()
+{
+    printf("--add + (rule num)        eg: --a 2}\n");
+    printf("--delete_all              eg: --delete_all}\n");
+    printf("--add_id + (rule id)      eg: --add_id 0 }\n");
+    printf("--delete_id + (rule id)   eg: --add_id 1 }\n");
+    printf("--loop                    eg: --loop }\n");
+    printf("--repeated                eg: --repeated }\n");
+}
+
+/* Rpc Init */
+static int rpc_init(struct rpc_instance * instance)
+{
+    char str_rpc_addr[MR_STRING_MAX] = { 0 };
+    unsigned int rpc_port = 0;
+
+    /* Get Ip & Port */
+    MESA_load_profile_string_def(GLOBAL_CONF_FILE_PATH, "rpc", "addr", str_rpc_addr, sizeof(str_rpc_addr), CTRLMSG_DEFAULT_ADDR);
+    MESA_load_profile_uint_def(GLOBAL_CONF_FILE_PATH, "rpc", "port", &rpc_port, CTRLMSG_DEFAULT_PORT);
+
+    /* Change Ip Addr */
+    struct sockaddr_in sockaddr_in;
+    if (inet_pton(AF_INET, str_rpc_addr, &sockaddr_in.sin_addr) <= 0)
+    {
+        return RT_ERR;
+    }
+
+    /* Set Port */
+    sockaddr_in.sin_port = htons(rpc_port);
+    sockaddr_in.sin_family = AF_INET;
+
+    /* Creat Msg Handler */
+    if(rpc_client_connect(instance->handler, sockaddr_in) != 0)
+    {
+        printf("Client Connect Server Failed !!!\n");
+        return RT_ERR;
+    }
+    return RT_SUCCESS;
+}
+
+void send_lb_rule_add_msg(struct rpc_instance * instance,int rule_id)
+{
+    int dev_num = 0;
+    char dev_str_section[MR_STRING_MAX];
+    char dev_name_str[MR_STRING_MAX];
+    cJSON *cjson_rule = NULL,* cjson_dev = NULL;
+
+    dev_num = rule_id%2 + 1;
+    cjson_rule = cJSON_CreateObject();
+
+    cJSON_AddNumberToObject(cjson_rule, "Id", rule_id);
+    cJSON_AddNumberToObject(cjson_rule, "Mode", 1);
+    cJSON_AddNumberToObject(cjson_rule, "DevNum", dev_num);
+
+    for (int i = 0; i < dev_num; i++)
+    {
+        cjson_dev = cJSON_CreateObject();
+        snprintf(dev_str_section, sizeof(dev_str_section), "%s%d", "Dev", i);
+        snprintf(dev_name_str, sizeof(dev_name_str), "%s%d", "nf_", i);
+        cJSON_AddStringToObject(cjson_dev, "DevName", dev_name_str);
+        cJSON_AddNumberToObject(cjson_dev, "DevType", 0);
+        cJSON_AddItemToObject(cjson_rule,dev_str_section,cjson_dev);
+    }
+
+    char *req_str = cJSON_Print(cjson_rule);
+    printf("%s\n",req_str);
+
+    if(rpc_client_send_request_async(instance->handler, MR_LB_TOPIC_SINGLE_RULE_ADD, cjson_rule) != 0)
+    {
+        printf("Send Add Request Failed !!!\n");
+    }
+    else
+    {
+        /* Set Creat Response Disable */
+        __atomic_exchange_n(&instance->response_flag, MR_CLI_RESPONSE_DISABLE, __ATOMIC_SEQ_CST);
+    }
+    free(req_str);
+    cJSON_Delete(cjson_rule);
+}
+
+void send_lb_rule_del_msg(struct rpc_instance * instance,int rule_id)
+{
+    cJSON *cjson_rule = NULL;
+
+    cjson_rule = cJSON_CreateObject();
+
+    cJSON_AddNumberToObject(cjson_rule, "RuleType", 4);
+    cJSON_AddNumberToObject(cjson_rule, "RuleId", rule_id);
+    cJSON_AddStringToObject(cjson_rule, "SrcIp", "172.16.1.10");
+    cJSON_AddNumberToObject(cjson_rule, "SrcIpMask", 24);
+    cJSON_AddStringToObject(cjson_rule, "DstIp", "172.16.1.100");
+    cJSON_AddNumberToObject(cjson_rule, "DstIpMask", 24);
+    cJSON_AddNumberToObject(cjson_rule, "SrcPortLow", 10);
+    cJSON_AddNumberToObject(cjson_rule, "SrcPortHigh", 60000);
+    cJSON_AddNumberToObject(cjson_rule, "DstPortLow", 20);
+    cJSON_AddNumberToObject(cjson_rule, "DstPortHigh", 60000);
+    cJSON_AddNumberToObject(cjson_rule, "Proto", IPPROTO_TCP);
+    cJSON_AddNumberToObject(cjson_rule, "Priority", 1);
+    cJSON_AddNumberToObject(cjson_rule, "SiId", rule_id);
+    cJSON_AddStringToObject(cjson_rule, "Action", "nf_steering");
+    cJSON_AddNumberToObject(cjson_rule, "NextGroup", rule_id%2);
+
+    if(rpc_client_send_request_async(instance->handler, MR_LB_TOPIC_SINGLE_RULE_DEL, cjson_rule) != 0)
+    {
+        printf("Send Del Request Failed !!!\n");
+    }
+    else
+    {
+        /* Set Creat Response Disable */
+        __atomic_exchange_n(&instance->response_flag, MR_CLI_RESPONSE_DISABLE, __ATOMIC_SEQ_CST);
+    }
+    cJSON_Delete(cjson_rule);
+}
+
+/* Rual Add Response */
+static int __rule_add_response_handler(cJSON *rsp, void *arg)
+{
+    cJSON * j_result = NULL;
+    struct rpc_instance * rpc_instance = (struct rpc_instance *)arg;
+
+    j_result = cJSON_GetObjectItem(rsp,MR_LB_RPC_RESULT);
+
+    if (j_result == NULL)
+    {
+        printf("The Result Is Null !\n");
+    }
+    else
+    {
+        if ((u_int32_t)j_result->valuedouble == RT_SUCCESS)
+        {
+            printf("The One LB Rule Add Sucess.\n");
+        }
+        else
+        {
+            printf("One LB Rule Add Error. Error Code:%u\n",(u_int32_t)j_result->valuedouble);
+        }
+    }
+
+    /* Set Creat Response Enable */
+    __atomic_exchange_n(&rpc_instance->response_flag, MR_CLI_RESPONSE_ENABLE, __ATOMIC_SEQ_CST);
+    return RT_SUCCESS;
+}
+
+/* Rual Delete Response */
+static int __rule_del_response_handler(cJSON *rsp, void *arg)
+{
+    cJSON * j_result = NULL;
+    struct rpc_instance * rpc_instance = (struct rpc_instance *)arg;
+
+    j_result = cJSON_GetObjectItem(rsp,MR_LB_RPC_RESULT);
+
+    if (j_result == NULL)
+    {
+        printf("The Result Is Null !\n");
+    }
+    else
+    {
+        if ((u_int32_t)j_result->valuedouble == RT_SUCCESS)
+        {
+            printf("The One LB Rule Del Sucess.\n");
+        }
+        else
+        {
+            printf("One LB Rule Del Error. Error Code:%d\n",(u_int32_t)j_result->valuedouble);
+        }
+    }
+
+    /* Set Creat Response Enable */
+    __atomic_exchange_n(&rpc_instance->response_flag, MR_CLI_RESPONSE_ENABLE, __ATOMIC_SEQ_CST);
+    return RT_SUCCESS;
+}
+
+int main(int argc, char * argv[])
+{
+    int opt = 0,type = 0;
+    uint32_t deal_num = 1024;
+    char * endptr = NULL;
+
+    while ((opt = getopt_long(argc, argv, MR_CLI_ARG_LIST, cli_options, NULL)) != -1)
+    {
+        switch (opt)
+        {
+            case 'a':
+            {
+                type = 1;
+                deal_num = (uint32_t)strtoull(optarg, &endptr, 0);
+                break;
+            }
+            case 'd':
+            {
+                type = 2;
+                deal_num = (uint32_t)strtoull(optarg, &endptr, 0);
+                break;
+            }
+            case MR_CLI_DELETE_ALL:
+            {
+                type = 2;
+                deal_num = 1024;
+                break;
+            }
+            case MR_CLI_ADD_FOR_ID:
+            {
+                type = 3;
+                deal_num = (uint32_t)strtoull(optarg, &endptr, 0);
+                break;
+            }
+            case MR_CLI_DEL_FOR_ID:
+            {
+                type = 4;
+                deal_num = (uint32_t)strtoull(optarg, &endptr, 0);
+                break;
+            }
+            case MR_CLI_LOOP:
+            {
+                type = 5;
+                break;
+            }
+            case MR_CLI_REPEATED:
+            {
+                type = 6;
+                break;
+            }
+
+            default:
+                help();
+                return 0;
+                break;
+        }
+    }
+
+    /* Create instance */
+    rpc_instance = rpc_create();
+    if (rpc_instance == NULL)
+    {
+        fprintf(stderr, "LB Rult Test instance create failed. ");
+        return 0;
+    }
+
+    if(rpc_client_recv_response_register(rpc_instance->handler, MR_LB_TOPIC_SINGLE_RULE_ADD, __rule_add_response_handler , rpc_instance) != 0)
+    {
+        printf("Register Add Response Callback Failed !!!\n");
+        return 0;
+    }
+
+    /*
+    if(rpc_client_recv_response_register(rpc_instance->handler, MR_LB_TOPIC_SINGLE_RULE_DEL, __rule_del_response_handler , rpc_instance) != 0)
+    {
+        printf("Register Del Response Callback Failed !!!\n");
+        return 0;
+    }
+    */
+
+    if (rpc_init(rpc_instance) != RT_SUCCESS)
+    {
+        fprintf(stderr, "Rpc module initialization failed. ");
+        return 0;
+    }
+
+    if(rpc_client_dispatch_thread(rpc_instance->handler) != 0)
+    {
+        printf("Client Start Dispach Thread Failed !!!\n");
+        return 0;
+    }
+
+    /* Send Requeset Msg */
+    if (type == 1)
+    {
+        /* Send Create Requeset Msg */
+        for (int i = 0; i < deal_num; i ++)
+        {
+            send_lb_rule_add_msg(rpc_instance,i);
+            printf("Send Add:%d\n",i);
+            while (__atomic_load_n(&rpc_instance->response_flag, __ATOMIC_RELAXED) == MR_CLI_RESPONSE_DISABLE)
+            {
+                usleep(1);
+            }
+        }
+    }
+    else if (type == 2)
+    {
+        for (int i = (deal_num - 1); i >= 0; i --)
+        {
+            send_lb_rule_del_msg(rpc_instance,i);
+            printf("Send Delete:%d\n",i);
+            while (__atomic_load_n(&rpc_instance->response_flag, __ATOMIC_RELAXED) == MR_CLI_RESPONSE_DISABLE)
+            {
+                usleep(1);
+            }
+        }
+    }
+    else if (type == 3)
+    {
+        send_lb_rule_add_msg(rpc_instance,deal_num);
+        printf("Send Add:%d\n",deal_num);
+        while (__atomic_load_n(&rpc_instance->response_flag, __ATOMIC_RELAXED) == MR_CLI_RESPONSE_DISABLE)
+        {
+            usleep(1);
+        }
+    }
+    else if (type == 4)
+    {
+        send_lb_rule_del_msg(rpc_instance,deal_num);
+        printf("Send Delete:%d\n",deal_num);
+        while (__atomic_load_n(&rpc_instance->response_flag, __ATOMIC_RELAXED) == MR_CLI_RESPONSE_DISABLE)
+        {
+            usleep(1);
+        }
+    }
+    else if (type == 5)
+    {
+        while (1)
+        {
+            /* Send Create Requeset Msg */
+            for (int i = 0; i < 2; i ++)
+            {
+                send_lb_rule_add_msg(rpc_instance,i);
+                printf("Send Add:%d\n",i);
+                while (__atomic_load_n(&rpc_instance->response_flag, __ATOMIC_RELAXED) == MR_CLI_RESPONSE_DISABLE)
+                {
+                    usleep(1);
+                }
+            }
+
+            for (int i = 1; i >= 0; i --)
+            {
+                send_lb_rule_del_msg(rpc_instance,i);
+                printf("Send Delete:%d\n",i);
+                while (__atomic_load_n(&rpc_instance->response_flag, __ATOMIC_RELAXED) == MR_CLI_RESPONSE_DISABLE)
+                {
+                    usleep(1);
+                }
+            }
+        }
+    }
+    else if (type == 6)
+    {
+        /* Send Create Requeset Msg */
+        for (int i = 0; i < 2; i ++)
+        {
+            send_lb_rule_add_msg(rpc_instance,0);
+            printf("Send Add:%d\n",i);
+            while (__atomic_load_n(&rpc_instance->response_flag, __ATOMIC_RELAXED) == MR_CLI_RESPONSE_DISABLE)
+            {
+                usleep(1);
+            }
+        }
+    }
+    else
+    {
+        help();
+    }
+
+    return RT_SUCCESS;
+}
\ No newline at end of file