🎈 perf: Refine counting structure in tera node.

 Refine counting structure in tera node.

3 files changed, 101 insertions, 71 deletions

diff --git a/service/src/node_bridge.c b/service/src/node_bridge.c
index 02a465d..9a63639 100644
--- a/service/src/node_bridge.c
+++ b/service/src/node_bridge.c
@@ -27,11 +27,7 @@ enum bridge_match_result
 };
 
 /* Bridge pkt match results string */
-static const char * bridge_match_result_str[] = {[BRIDGE_MATCH_MISS] = "miss",
-                                                 [BRIDGE_MATCH_TO_MASTER] = "to master",
-                                                 [BRIDGE_MATCH_BROADCAST] = "broadcast",
-                                                 [BRIDGE_MATCH_DST_MAC] = "match dst mac",
-                                                 [BRIDGE_MATCH_MAX] = "unknown"};
+static const char * bridge_match_result_str[BRIDGE_MATCH_MAX] = {"miss", "to master", "broadcast", "match dst mac"};
 
 /* Bridge stats struct */
 struct bridge_stats
@@ -231,6 +227,8 @@ static __rte_always_inline void gen_store_trace_info_original(struct rte_node *
 {
     /* Populate the next node infomation */
     char str_record[MR_STRING_MAX];
+
+    assert(match_result < BRIDGE_MATCH_MAX);
     int len = snprintf(str_record, sizeof(str_record), ", rsn:%s", bridge_match_result_str[match_result]);
 
     /* Populate the egress port information */
diff --git a/service/src/node_etherfabric.c b/service/src/node_etherfabric.c
index 60c2276..16b5c94 100644
--- a/service/src/node_etherfabric.c
+++ b/service/src/node_etherfabric.c
@@ -86,7 +86,7 @@ enum ef_ingress_drop_reason
 };
 
 /* Etherfabric ingress drop reason string */
-char * ef_ingr_drop_rsn_str[] = {"adapter lookup miss", "sid prepend err", "ef peer lookup miss", "unknown"};
+char * ef_ingr_drop_rsn_str[EF_INGR_DROP_RSN_MAX] = {"adapter lookup miss", "sid prepend err", "ef peer lookup miss"};
 
 /* Etherfabric ingress stats struct */
 struct ef_ingress_stats
@@ -106,7 +106,7 @@ enum ef_egress_drop_reason
 };
 
 /* Etherfabric egress drop reason string */
-char * ef_egr_drop_rsn_str[] = {"tap mode", "invalid mode", "unknown"};
+char * ef_egr_drop_rsn_str[EF_EGR_DROP_RSN_MAX] = {"tap mode", "invalid mode"};
 
 /* Etherfabric egress stats struct */
 struct ef_egress_stats
@@ -493,6 +493,7 @@ static __rte_always_inline void gen_store_trace_info_ingress(struct rte_node * n
     /* Populate the reason for next node */
     if (unlikely(next_node_index == EF_INGRESS_NEXT_PKT_DROP))
     {
+        assert(drop_reason < EF_INGR_DROP_RSN_MAX);
         len += snprintf(str_record + len, sizeof(str_record) - len, ", rsn:%s", ef_ingr_drop_rsn_str[drop_reason]);
     }
     else
@@ -745,6 +746,7 @@ static __rte_always_inline void gen_store_trace_info_egress(struct rte_node * no
 
     if (unlikely(next_node_index == EF_EGRESS_NEXT_PKT_DROP))
     {
+        assert(drop_reason < EF_EGR_DROP_RSN_MAX);
         len += snprintf(str_record + len, sizeof(str_record) - len, ", rsn:%s", ef_egr_drop_rsn_str[drop_reason]);
     }
     else
diff --git a/service/src/node_tera.c b/service/src/node_tera.c
index b4fba43..aeda4cd 100644
--- a/service/src/node_tera.c
+++ b/service/src/node_tera.c
@@ -34,19 +34,31 @@ struct tera_adapter
     struct mr_dev_desc * listen_device;
 };
 
+/* Tera ingress drop reason */
+enum tera_ingress_drop_reason
+{
+    TERA_INGR_DROP_RSN_NO_MATCH = 0,
+    TERA_INGR_DROP_RSN_APPEND_SID_ERR,
+    TERA_INGR_DROP_RSN_MAX
+};
+
+/* Tera ingress drop reason string */
+static const char * tera_ingress_drop_reason_str[TERA_INGR_DROP_RSN_MAX] = {"adapter lookup", "append sid err"};
+
 /* Tera ingress stats struct */
 struct tera_ingress_stats
 {
     volatile uint64_t deal_pkts;
-    volatile uint64_t drop_for_no_match;
-    volatile uint64_t drop_for_append_sid_err;
+    volatile uint64_t pkts_per_batch;
     volatile uint64_t to_classifier;
+    volatile uint64_t drop_reason[TERA_INGR_DROP_RSN_MAX];
 } __rte_cache_aligned;
 
 /* Tera egress stats struct */
 struct tera_egress_stats
 {
     volatile uint64_t deal_pkts;
+    volatile uint64_t pkts_per_batch;
 } __rte_cache_aligned;
 
 /* Tera main struct */
@@ -311,13 +323,11 @@ static inline struct tera_adapter * tera_adapter_match(const struct node_tera_ma
 /* Generate and store the trace information for ingress */
 static __rte_always_inline void gen_store_trace_info_ingress(struct rte_node * node, struct rte_mbuf * mbuf,
                                                              uint16_t next_node_index,
-                                                             struct tera_ingress_stats * stats,
-                                                             struct tera_ingress_stats * stats_for_trace,
-                                                             uint16_t vlan_id, uint16_t vlan_couple,
-                                                             uint8_t mac_flipping, uint16_t prepend_sid)
+                                                             struct tera_ingress_stats * stats, uint16_t vlan_id,
+                                                             uint16_t vlan_couple, uint8_t mac_flipping,
+                                                             uint16_t prepend_sid,
+                                                             enum tera_ingress_drop_reason drop_reason)
 {
-    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
-
     /* Populate the next node infomation */
     char str_record[MR_STRING_MAX];
     int len = snprintf(str_record, sizeof(str_record), "next node:%s", node->nodes[next_node_index]->name);
@@ -325,41 +335,23 @@ static __rte_always_inline void gen_store_trace_info_ingress(struct rte_node * n
     /* Populate the drop reason */
     if (unlikely(next_node_index == TERA_INGRESS_NEXT_PKT_DROP))
     {
-        if (stats_for_trace->drop_for_no_match != stats->drop_for_no_match)
-        {
-            stats_for_trace->drop_for_no_match = stats->drop_for_no_match;
-            len += snprintf(str_record + len, sizeof(str_record) - len, ", rsn:adapter lookup");
-        }
-        else if (stats_for_trace->drop_for_append_sid_err != stats->drop_for_append_sid_err)
-        {
-            stats_for_trace->drop_for_append_sid_err = stats->drop_for_append_sid_err;
-            len += snprintf(str_record + len, sizeof(str_record) - len, ", rsn:append sid err");
-        }
+        assert(drop_reason < TERA_INGR_DROP_RSN_MAX);
+        len +=
+            snprintf(str_record + len, sizeof(str_record) - len, ", rsn:%s", tera_ingress_drop_reason_str[drop_reason]);
     }
     else
     {
-        /* Populate the tera id */
         struct mrb_metadata * mrb_meta = mrbuf_cz_data(mbuf, MR_NODE_CTRLZONE_ID);
-        len += snprintf(str_record + len, sizeof(str_record) - len, ", tera id:%u", mrb_meta->adapter_id);
-
-        /* Populate the dir */
-        len += snprintf(str_record + len, sizeof(str_record) - len, ", dir:%u", mrb_meta->dir);
-
-        /* Populate the vlan modification details */
-        len +=
-            snprintf(str_record + len, sizeof(str_record) - len, ", vlan:%u to %u", ntohs(vlan_id), ntohs(vlan_couple));
-
-        /* Populate the mac flipping details */
-        len += snprintf(str_record + len, sizeof(str_record) - len, ", mac flipping:%u", mac_flipping);
-
-        /* Populate the prepend sid */
-        len += snprintf(str_record + len, sizeof(str_record) - len, ", prep sid:%u", prepend_sid);
+        len += snprintf(str_record + len, sizeof(str_record) - len,
+                        ", tera id:%u, dir:%u, vlan:%u to %u, mac flipping:%u, prep sid:%u", mrb_meta->adapter_id,
+                        mrb_meta->dir, ntohs(vlan_id), ntohs(vlan_couple), mac_flipping, prepend_sid);
 
         /* Populate the sids information */
         len += embed_sid_info(mbuf, str_record + len, sizeof(str_record) - len);
     }
 
     /* Emit the trace record */
+    struct dp_trace_record_meta meta = {.appsym = MR_TRACE_APPSYM, .module = node->name};
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
 }
 
@@ -379,8 +371,8 @@ static __rte_always_inline uint16_t tera_ingress_node_process(struct rte_graph *
     uint16_t prev_port_id = UINT16_MAX;
     uint16_t prev_vlan_id = UINT16_MAX;
     uint16_t prev_vlan_couple = UINT16_MAX;
+    enum tera_ingress_drop_reason drop_reason = TERA_INGR_DROP_RSN_MAX;
     struct tera_ingress_stats stats = {};
-    struct tera_ingress_stats stats_for_trace = {};
     struct tera_adapter * prev_adapter = NULL;
     struct node_tera_main * tera_main = node_tera_main_get();
 
@@ -419,7 +411,8 @@ static __rte_always_inline uint16_t tera_ingress_node_process(struct rte_graph *
         uint16_t next_node_index;
         if (prev_adapter == NULL)
         {
-            stats.drop_for_no_match++;
+            drop_reason = TERA_INGR_DROP_RSN_NO_MATCH;
+            stats.drop_reason[drop_reason]++;
             next_node_index = TERA_INGRESS_NEXT_PKT_DROP;
             goto node_enqueue;
         }
@@ -454,7 +447,8 @@ static __rte_always_inline uint16_t tera_ingress_node_process(struct rte_graph *
         prepend_sid = tera_main->sid_start + prev_adapter->tera_adapter_id;
         if (unlikely(sid_list_prepend(&mrb_meta->sid_list, &prepend_sid, 1) == RT_ERR))
         {
-            stats.drop_for_append_sid_err++;
+            drop_reason = TERA_INGR_DROP_RSN_APPEND_SID_ERR;
+            stats.drop_reason[drop_reason]++;
             next_node_index = TERA_INGRESS_NEXT_PKT_DROP;
             goto node_enqueue;
         }
@@ -467,8 +461,8 @@ static __rte_always_inline uint16_t tera_ingress_node_process(struct rte_graph *
         /* Check if tracing is enabled for the current Mbuf */
         if (unlikely(dp_trace_record_can_emit(mbuf)))
         {
-            gen_store_trace_info_ingress(node, mbuf, next_node_index, &stats, &stats_for_trace, vlan_id,
-                                         prev_vlan_couple, prev_adapter->mac_flipping, prepend_sid);
+            gen_store_trace_info_ingress(node, mbuf, next_node_index, &stats, vlan_id, prev_vlan_couple,
+                                         prev_adapter->mac_flipping, prepend_sid, drop_reason);
             // gen_store_trace_info_sid_list(node, mbuf);
         }
 
@@ -495,10 +489,14 @@ static __rte_always_inline uint16_t tera_ingress_node_process(struct rte_graph *
     /* Update graph stats */
     struct tera_ingress_stats * graph_stats = &ingress_stats_per_graph[graph->id];
     graph_stats->deal_pkts += cnt;
-    graph_stats->drop_for_no_match += stats.drop_for_no_match;
-    graph_stats->drop_for_append_sid_err += stats.drop_for_append_sid_err;
+    graph_stats->pkts_per_batch = cnt;
     graph_stats->to_classifier += stats.to_classifier;
 
+    for (int i = 0; i < TERA_INGR_DROP_RSN_MAX; i++)
+    {
+        graph_stats->drop_reason[i] += stats.drop_reason[i];
+    }
+
     return cnt;
 }
 
@@ -525,14 +523,10 @@ static __rte_always_inline void gen_store_trace_info_egress(struct rte_node * no
 
     /* Populate the next node infomation */
     char str_record[MR_STRING_MAX];
-    int len = snprintf(str_record, sizeof(str_record), "next node:%s", node->nodes[next_node_index]->name);
-
-    /* Populate the adapter id */
     struct mrb_metadata * mrb_meta = mrbuf_cz_data(mbuf, MR_NODE_CTRLZONE_ID);
-    len += snprintf(str_record + len, sizeof(str_record) - len, ", tera id:%u", mrb_meta->adapter_id);
 
-    /* Populate the egress port info */
-    len += snprintf(str_record + len, sizeof(str_record) - len, ", tx:%u", mrb_meta->port_egress);
+    snprintf(str_record, sizeof(str_record), "next node:%s, tera id:%u, tx:%u", node->nodes[next_node_index]->name,
+             mrb_meta->adapter_id, mrb_meta->port_egress);
 
     /* Emit the trace record */
     dp_trace_record_emit_str(sc_main_get()->trace, mbuf, rte_lcore_id(), &meta, str_record);
@@ -594,6 +588,7 @@ static __rte_always_inline uint16_t tera_egress_node_process(struct rte_graph *
     /* Update graph stats */
     struct tera_egress_stats * graph_stats = &egress_stats_per_graph[graph->id];
     graph_stats->deal_pkts += cnt;
+    graph_stats->pkts_per_batch = cnt;
 
     /* Move to next node */
     rte_node_next_stream_move(graph, node, TERA_EGRESS_NEXT_ETH_EGRESS);
@@ -619,48 +614,83 @@ RTE_NODE_REGISTER(tera_egress_node_base);
 cJSON * tera_ingress_node_monit_loop(struct sc_main * sc)
 {
     cJSON * json_root = cJSON_CreateObject();
-    for (uint32_t graph_id = 0; graph_id < sc->nr_io_thread; graph_id++)
+    unsigned int nr_graphs = sc->nr_io_thread;
+
+    uint64_t total_pkts[nr_graphs];
+    uint64_t pkts_per_batch[nr_graphs];
+    uint64_t to_classifier[nr_graphs];
+    uint64_t drop_reason[nr_graphs][TERA_INGR_DROP_RSN_MAX];
+
+    for (uint32_t graph_id = 0; graph_id < nr_graphs; graph_id++)
     {
         struct tera_ingress_stats * stats = &ingress_stats_per_graph[graph_id];
         if (stats->deal_pkts == 0)
         {
+            total_pkts[graph_id] = 0;
+            pkts_per_batch[graph_id] = 0;
+            to_classifier[graph_id] = 0;
+
+            memset(drop_reason[graph_id], 0, sizeof(drop_reason[graph_id]));
             continue;
         }
 
-        cJSON * graph_obj = cJSON_CreateObject();
-        cJSON_AddNumberToObject(graph_obj, "deal_pkts", stats->deal_pkts);
-        cJSON_AddNumberToObject(graph_obj, "drop_for_no_match", stats->drop_for_no_match);
-        cJSON_AddNumberToObject(graph_obj, "drop_for_append_sid_err", stats->drop_for_append_sid_err);
-        cJSON_AddNumberToObject(graph_obj, "to_classifier", stats->to_classifier);
+        total_pkts[graph_id] = stats->deal_pkts;
+        pkts_per_batch[graph_id] = stats->pkts_per_batch;
+        to_classifier[graph_id] = stats->to_classifier;
 
-        char str_graph_idx[MR_STRING_MAX];
-        snprintf(str_graph_idx, sizeof(str_graph_idx) - 1, "graph-%u", graph_id);
-        cJSON_AddItemToObject(json_root, str_graph_idx, graph_obj);
+        for (int i = 0; i < TERA_INGR_DROP_RSN_MAX; i++)
+        {
+            drop_reason[graph_id][i] = stats->drop_reason[i];
+        }
+    }
+
+    cJSON * json_total_pkts = create_uint64_array(total_pkts, nr_graphs);
+    cJSON_AddItemToObject(json_root, "tera ingress, total pkts", json_total_pkts);
+
+    cJSON * json_pkts_per_batch = create_uint64_array(pkts_per_batch, nr_graphs);
+    cJSON_AddItemToObject(json_root, "tera ingress, pkts per batch", json_pkts_per_batch);
+
+    cJSON * json_to_classifier = create_uint64_array(to_classifier, nr_graphs);
+    cJSON_AddItemToObject(json_root, "tera ingress, to classifier", json_to_classifier);
+
+    for (int i = 0; i < TERA_INGR_DROP_RSN_MAX; i++)
+    {
+        char str_title[MR_STRING_MAX];
+        snprintf(str_title, sizeof(str_title), "tera ingress, %s", tera_ingress_drop_reason_str[i]);
+
+        cJSON * json_drop_reason = create_uint64_array(drop_reason[i], nr_graphs);
+        cJSON_AddItemToObject(json_root, str_title, json_drop_reason);
     }
 
-    cJSON_AddNumberToObject(json_root, "nr_graph_total", sc->nr_io_thread);
     return json_root;
 }
 
 cJSON * tera_egress_node_monit_loop(struct sc_main * sc)
 {
     cJSON * json_root = cJSON_CreateObject();
-    for (uint32_t graph_id = 0; graph_id < sc->nr_io_thread; graph_id++)
+    unsigned int nr_graphs = sc->nr_io_thread;
+
+    uint64_t total_pkts[nr_graphs];
+    uint64_t pkts_per_batch[nr_graphs];
+
+    for (uint32_t graph_id = 0; graph_id < nr_graphs; graph_id++)
     {
         struct tera_egress_stats * stats = &egress_stats_per_graph[graph_id];
         if (stats->deal_pkts == 0)
         {
+            total_pkts[graph_id] = 0;
+            pkts_per_batch[graph_id] = 0;
             continue;
         }
 
-        cJSON * graph_obj = cJSON_CreateObject();
-        cJSON_AddNumberToObject(graph_obj, "deal_pkts", stats->deal_pkts);
-
-        char str_graph_idx[MR_STRING_MAX];
-        snprintf(str_graph_idx, sizeof(str_graph_idx) - 1, "graph-%u", graph_id);
-        cJSON_AddItemToObject(json_root, str_graph_idx, graph_obj);
+        total_pkts[graph_id] = stats->deal_pkts;
+        pkts_per_batch[graph_id] = stats->pkts_per_batch;
     }
 
-    cJSON_AddNumberToObject(json_root, "nr_graph_total", sc->nr_io_thread);
+    cJSON * json_total_pkts = create_uint64_array(total_pkts, nr_graphs);
+    cJSON_AddItemToObject(json_root, "tera egress, total pkts", json_total_pkts);
+
+    cJSON * json_pkts_per_batch = create_uint64_array(pkts_per_batch, nr_graphs);
+    cJSON_AddItemToObject(json_root, "tera egress, pkts per batch", json_pkts_per_batch);
     return json_root;
 }